diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java index 959f37d594d51..5a1bf489beab8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java @@ -30,7 +30,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; import org.apache.iotdb.confignode.rpc.thrift.TSetDataReplicationFactorReq; import org.apache.iotdb.confignode.rpc.thrift.TSetSchemaReplicationFactorReq; -import org.apache.iotdb.confignode.rpc.thrift.TSetTimePartitionIntervalReq; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -138,9 +137,6 @@ public void testSetAndQueryDatabase() throws Exception { Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); status = client.setDataReplicationFactor(new TSetDataReplicationFactorReq(sg1, 1)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - status = client.setTimePartitionInterval(new TSetTimePartitionIntervalReq(sg1, 604800L)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - // test setter results getResp = client.getMatchedDatabaseSchemas( @@ -154,7 +150,7 @@ public void testSetAndQueryDatabase() throws Exception { Assert.assertEquals(sg1, databaseSchema.getName()); Assert.assertEquals(1, databaseSchema.getSchemaReplicationFactor()); Assert.assertEquals(1, databaseSchema.getDataReplicationFactor()); - Assert.assertEquals(604800, databaseSchema.getTimePartitionInterval()); + Assert.assertEquals(2048L, databaseSchema.getTimePartitionInterval()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java index 470e68de65153..82a23d09d02d3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java @@ -28,6 +28,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -44,12 +45,14 @@ import java.io.IOException; import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @RunWith(IoTDBTestRunner.class) @@ -142,9 +145,83 @@ public void testTimePartition() throws Exception { } timestatmps.forEach( t -> { - long timePartitionId = TimePartitionUtils.getTimePartitionId(t); + long timePartitionId = TimePartitionUtils.getTimePartitionId(t, "root.sg1"); assertTrue(timePartitions.contains(timePartitionId)); }); } } + + @Test + public void testDefaultTimePartitionConfigAfterClusterRestart() throws Exception { + final String database = "root.default_time_partition"; + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + database); + + final SQLException originException = + assertThrows( + SQLException.class, + () -> + statement.execute( + "ALTER DATABASE " + database + " WITH TIME_PARTITION_ORIGIN=0")); + assertTrue( + originException.getMessage(), + originException.getMessage().contains("Doesn't support ALTER TimePartitionOrigin yet.")); + + final SQLException intervalException = + assertThrows( + SQLException.class, + () -> + statement.execute( + "ALTER DATABASE " + database + " WITH TIME_PARTITION_INTERVAL=1")); + assertTrue( + intervalException.getMessage(), + intervalException + .getMessage() + .contains("Doesn't support ALTER TimePartitionInterval yet.")); + } + + assertDefaultTimePartitionConfig(database); + TestUtils.restartCluster(EnvFactory.getEnv()); + assertDefaultTimePartitionConfig(database); + + final long[] timestamps = {0L, 3601000L, 7201000L}; + for (int i = 0; i < timestamps.length; i++) { + try (final Connection connection = + EnvFactory.getEnv().getConnection(EnvFactory.getEnv().getDataNodeWrapper(i)); + final Statement statement = connection.createStatement()) { + statement.execute( + "INSERT INTO " + database + ".d1(timestamp, s1) values(" + timestamps[i] + ", 1)"); + } + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + final List timePartitions = new ArrayList<>(); + try (final ResultSet result = + statement.executeQuery("SHOW TIMEPARTITION WHERE DATABASE = " + database)) { + while (result.next()) { + timePartitions.add(result.getLong(ColumnHeaderConstant.TIME_PARTITION)); + } + } + assertTrue(timePartitions.contains(-1L)); + assertTrue(timePartitions.contains(1L)); + assertTrue(timePartitions.contains(2L)); + } + } + + private void assertDefaultTimePartitionConfig(final String database) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + final TShowDatabaseResp response = client.showDatabase(showAllDatabasesReq); + assertTrue(response.databaseInfoMap.containsKey(database)); + assertEquals( + TEST_TIME_PARTITION_ORIGIN, + response.databaseInfoMap.get(database).getTimePartitionOrigin()); + assertEquals( + TEST_TIME_PARTITION_INTERVAL, + response.databaseInfoMap.get(database).getTimePartitionInterval()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index cdb7254b8c7a5..7221c0ac0280f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -119,6 +119,7 @@ public void testManageDatabase() { String[] TTLs = new String[] {"INF"}; int[] schemaReplicaFactors = new int[] {1}; int[] dataReplicaFactors = new int[] {1}; + long[] timePartitionOrigin = new long[] {0L}; int[] timePartitionInterval = new int[] {604800000}; // show @@ -167,12 +168,13 @@ public void testManageDatabase() { assertEquals(TTLs[cnt], resultSet.getString(2)); assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(3)); assertEquals(dataReplicaFactors[cnt], resultSet.getInt(4)); - assertEquals(timePartitionInterval[cnt], resultSet.getLong(5)); - assertEquals(schemaRegionGroupNum[cnt], resultSet.getInt(6)); - assertTrue(resultSet.getInt(7) >= defaultSchemaRegionGroupNum[cnt]); - assertEquals(dataRegionGroupNum[cnt], resultSet.getInt(8)); - assertTrue(resultSet.getInt(9) >= defaultDataRegionGroupNum[cnt]); - assertFalse(resultSet.getBoolean(10)); + assertEquals(timePartitionOrigin[cnt], resultSet.getLong(5)); + assertEquals(timePartitionInterval[cnt], resultSet.getLong(6)); + assertEquals(schemaRegionGroupNum[cnt], resultSet.getInt(7)); + assertTrue(resultSet.getInt(8) >= defaultSchemaRegionGroupNum[cnt]); + assertEquals(dataRegionGroupNum[cnt], resultSet.getInt(9)); + assertTrue(resultSet.getInt(10) >= defaultDataRegionGroupNum[cnt]); + assertFalse(resultSet.getBoolean(11)); cnt++; } assertEquals(databaseNames.length, cnt); @@ -209,9 +211,11 @@ public void testManageDatabase() { statement.execute("drop database IF EXISTS test"); // Test create database with properties - statement.execute("create database test_prop with (ttl=300, time_partition_interval=100000)"); + statement.execute( + "create database test_prop with (ttl=300, time_partition_origin=2000, time_partition_interval=100000)"); databaseNames = new String[] {"test_prop"}; TTLs = new String[] {"300"}; + timePartitionOrigin = new long[] {2000L}; timePartitionInterval = new int[] {100000}; // show @@ -236,6 +240,28 @@ public void testManageDatabase() { assertEquals(databaseNames.length, cnt); } + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) { + int cnt = 0; + while (resultSet.next()) { + if (resultSet.getString(1).equals("information_schema")) { + continue; + } + assertEquals(databaseNames[cnt], resultSet.getString(1)); + assertEquals(TTLs[cnt], resultSet.getString(2)); + assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(3)); + assertEquals(dataReplicaFactors[cnt], resultSet.getInt(4)); + assertEquals(timePartitionOrigin[cnt], resultSet.getLong(5)); + assertEquals(timePartitionInterval[cnt], resultSet.getLong(6)); + assertEquals(schemaRegionGroupNum[cnt], resultSet.getInt(7)); + assertTrue(resultSet.getInt(8) >= defaultSchemaRegionGroupNum[cnt]); + assertEquals(dataRegionGroupNum[cnt], resultSet.getInt(9)); + assertTrue(resultSet.getInt(10) >= defaultDataRegionGroupNum[cnt]); + assertTrue(resultSet.getBoolean(11)); + cnt++; + } + assertEquals(databaseNames.length, cnt); + } + try { statement.execute("create database test_prop_2 with (non_exist_prop=DEFAULT)"); fail( @@ -304,7 +330,7 @@ public void testShowCreateDatabase() throws SQLException { EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( - "create database test_show_create_db with (ttl=300, max_schema_region_group_num=DEFAULT, max_data_region_group_num=DEFAULT, time_partition_interval=100000)"); + "create database test_show_create_db with (ttl=300, max_schema_region_group_num=DEFAULT, max_data_region_group_num=DEFAULT, time_partition_origin=2000, time_partition_interval=100000)"); try (final ResultSet resultSet = statement.executeQuery("show create database test_show_create_db")) { @@ -315,9 +341,11 @@ public void testShowCreateDatabase() throws SQLException { createDatabaseSQL, createDatabaseSQL.startsWith("CREATE DATABASE \"test_show_create_db\" WITH (")); assertTrue(createDatabaseSQL, createDatabaseSQL.contains("ttl=300")); + assertTrue(createDatabaseSQL, createDatabaseSQL.contains("time_partition_origin=2000")); assertTrue(createDatabaseSQL, createDatabaseSQL.contains("time_partition_interval=100000")); assertTrue(createDatabaseSQL, createDatabaseSQL.contains("max_schema_region_group_num=")); assertTrue(createDatabaseSQL, createDatabaseSQL.contains("max_data_region_group_num=")); + assertTrue(createDatabaseSQL, createDatabaseSQL.contains("need_last_cache=true")); assertFalse(resultSet.next()); } } @@ -559,6 +587,7 @@ public void testInformationSchema() throws SQLException { "ttl(ms),STRING,ATTRIBUTE,", "schema_replication_factor,INT32,ATTRIBUTE,", "data_replication_factor,INT32,ATTRIBUTE,", + "time_partition_origin,INT64,ATTRIBUTE,", "time_partition_interval,INT64,ATTRIBUTE,", "schema_region_group_num,INT32,ATTRIBUTE,", "max_schema_region_group_num,INT32,ATTRIBUTE,", @@ -867,21 +896,22 @@ public void testInformationSchema() throws SQLException { int cnt = 0; while (resultSet.next()) { if ("information_schema".equals(resultSet.getString(1))) { - for (int columnIndex = 3; columnIndex <= 9; columnIndex++) { + for (int columnIndex = 3; columnIndex < expectedColumnSchemas.size(); columnIndex++) { assertNull(resultSet.getObject(columnIndex)); } - assertFalse(resultSet.getBoolean(10)); + assertFalse(resultSet.getBoolean(11)); } else { assertEquals("test", resultSet.getString(1)); assertEquals("INF", resultSet.getString(2)); assertEquals(1, resultSet.getInt(3)); assertEquals(1, resultSet.getInt(4)); - assertEquals(604800000, resultSet.getLong(5)); - assertEquals(0, resultSet.getInt(6)); - assertTrue(resultSet.getInt(7) >= 1); - assertEquals(0, resultSet.getInt(8)); - assertTrue(resultSet.getInt(9) >= 2); - assertTrue(resultSet.getBoolean(10)); + assertEquals(0, resultSet.getLong(5)); + assertEquals(604800000, resultSet.getLong(6)); + assertEquals(0, resultSet.getInt(7)); + assertTrue(resultSet.getInt(8) >= 1); + assertEquals(0, resultSet.getInt(9)); + assertTrue(resultSet.getInt(10) >= 2); + assertTrue(resultSet.getBoolean(11)); } cnt++; } @@ -1155,9 +1185,9 @@ public void testDBAuth() throws SQLException { Collections.singleton("information_schema,INF,null,null,null,")); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.databases"), - "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,max_schema_region_group_num,data_region_group_num,max_data_region_group_num,need_last_cache,", + "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_origin,time_partition_interval,schema_region_group_num,max_schema_region_group_num,data_region_group_num,max_data_region_group_num,need_last_cache,", Collections.singleton( - "information_schema,INF,null,null,null,null,null,null,null,false,")); + "information_schema,INF,null,null,null,null,null,null,null,null,false,")); } try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 4424d31843a8a..7667d67e4b55a 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -122,6 +122,7 @@ databaseAttributeClause databaseAttributeKey : TTL | TIME_PARTITION_INTERVAL + | TIME_PARTITION_ORIGIN | MAX_SCHEMA_REGION_GROUP_NUM | MAX_DATA_REGION_GROUP_NUM ; diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 755b0a6d4d62f..555303d695b8f 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -1239,6 +1239,10 @@ TIME_PARTITION_INTERVAL : T I M E '_' P A R T I T I O N '_' I N T E R V A L ; +TIME_PARTITION_ORIGIN + : T I M E '_' P A R T I T I O N '_' O R I G I N + ; + MAX_SCHEMA_REGION_GROUP_NUM : M A X '_' S C H E M A '_' R E G I O N '_' G R O U P '_' N U M ; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index c1d8878ee9bd9..81f6b85fe3bdf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -49,6 +49,7 @@ public enum ConfigPhysicalPlanType { SetTTL((short) 201), SetSchemaReplicationFactor((short) 202), SetDataReplicationFactor((short) 203), + // Retained for compatibility with legacy consensus logs. No public API writes this plan. SetTimePartitionInterval((short) 204), AdjustMaxRegionGroupNum((short) 205), DeleteDatabase((short) 206), diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionGroupsByTimePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionGroupsByTimePlan.java index d946022771da5..4a7023c77af50 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionGroupsByTimePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionGroupsByTimePlan.java @@ -46,8 +46,8 @@ public GetRegionGroupsByTimePlan( final String database, final long startTime, final long endTime) { super(ConfigPhysicalPlanType.GetRegionGroupsByTime); this.database = database; - this.startTimeSlot = TimePartitionUtils.getTimePartitionSlot(startTime); - this.endTimeSlot = TimePartitionUtils.getTimePartitionSlot(endTime); + this.startTimeSlot = TimePartitionUtils.getTimePartitionSlot(startTime, database); + this.endTimeSlot = TimePartitionUtils.getTimePartitionSlot(endTime, database); } public String getDatabase() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/SetTimePartitionIntervalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/SetTimePartitionIntervalPlan.java index 69fcd910c651c..53dfd0b52c53f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/SetTimePartitionIntervalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/SetTimePartitionIntervalPlan.java @@ -28,6 +28,7 @@ import java.nio.ByteBuffer; import java.util.Objects; +/** Retained only for replaying consensus logs written before the interval became immutable. */ public class SetTimePartitionIntervalPlan extends ConfigPhysicalPlan { private String storageGroup; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index f82d8187fab33..a612b56945aff 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -96,7 +96,6 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetDataReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetSchemaReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; -import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.response.ainode.AINodeRegisterResp; @@ -745,17 +744,6 @@ public TSStatus setDataReplicationFactor( } } - @Override - public TSStatus setTimePartitionInterval( - SetTimePartitionIntervalPlan setTimePartitionIntervalPlan) { - TSStatus status = confirmLeader(); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return clusterSchemaManager.setTimePartitionInterval(setTimePartitionIntervalPlan); - } else { - return status; - } - } - @Override public DataSet countMatchedDatabases(CountDatabasePlan countDatabasePlan) { TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 37cc7c72247cc..ba81f8f2dd955 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -51,7 +51,6 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetDataReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetSchemaReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; -import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.cq.CQManager; @@ -394,8 +393,6 @@ public interface IManager { TSStatus setDataReplicationFactor(SetDataReplicationFactorPlan configPhysicalPlan); - TSStatus setTimePartitionInterval(SetTimePartitionIntervalPlan configPhysicalPlan); - /** * Count Databases. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index 748c8db9acfdf..ea5c7e18962b8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -892,6 +892,8 @@ private TSStatus executePlan(final ConfigPhysicalPlan plan) throws ConsensusExce ConfigNodeDescriptor.getInstance().getConf().getSchemaReplicationFactor()); schema.setDataReplicationFactor( ConfigNodeDescriptor.getInstance().getConf().getDataReplicationFactor()); + schema.setTimePartitionOrigin( + CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin()); schema.setTimePartitionInterval( CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()); schema.setMinSchemaRegionGroupNum( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index e7f96322a5346..edbb87746771d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -65,7 +65,6 @@ import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetDataReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetSchemaReplicationFactorPlan; -import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreAlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTableColumnCommentPlan; @@ -473,19 +472,6 @@ public TSStatus setDataReplicationFactor( } } - public TSStatus setTimePartitionInterval( - SetTimePartitionIntervalPlan setTimePartitionIntervalPlan) { - // TODO: Inform DataNodes - try { - return getConsensusManager().write(setTimePartitionIntervalPlan); - } catch (ConsensusException e) { - LOGGER.warn(CONSENSUS_WRITE_ERROR, e); - TSStatus result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - result.setMessage(e.getMessage()); - return result; - } - } - /** * Only leader use this interface. Adjust the maxSchemaRegionGroupNum and maxDataRegionGroupNum of * each Database based on existing cluster resources diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index 8919308a159c5..bee2fb5562979 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -434,6 +434,7 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return clusterSchemaInfo.setDataReplicationFactor( (SetDataReplicationFactorPlan) physicalPlan); case SetTimePartitionInterval: + // This type is no longer writable and remains solely for replaying legacy consensus logs. return clusterSchemaInfo.setTimePartitionInterval( (SetTimePartitionIntervalPlan) physicalPlan); case CreateRegionGroups: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java index 3e3170af19f6e..a2555363241af 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java @@ -648,7 +648,7 @@ public Map getLastDataAllotTable() { */ public void autoCleanPartitionTable(long TTL, TTimePartitionSlot currentTimeSlot) { long[] removedTimePartitionSlots = - dataPartitionTable.autoCleanPartitionTable(TTL, currentTimeSlot).stream() + dataPartitionTable.autoCleanPartitionTable(TTL, currentTimeSlot, databaseName).stream() .map(TTimePartitionSlot::getStartTime) .collect(Collectors.toList()) .stream() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 6bae27bc7f28a..b9306e52735cf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -39,6 +39,7 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; @@ -211,6 +212,8 @@ public TSStatus createDatabase(final DatabaseSchemaPlan plan) { .getDatabaseNodeByDatabasePath(partialPathName) .getAsMNode() .setDatabaseSchema(databaseSchema); + TimePartitionUtils.updateDatabaseTimePartitionConfig( + databaseSchema.getName(), databaseSchema); if (databaseSchema.isIsTableModel()) { configSchemaStatistics.increaseTableDatabaseNum(); @@ -239,6 +242,18 @@ public TSStatus alterDatabase(final DatabaseSchemaPlan plan) { databaseReadWriteLock.writeLock().lock(); try { final TDatabaseSchema alterSchema = plan.getSchema(); + if (alterSchema.isSetTimePartitionOrigin()) { + return new TSStatus(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode()) + .setMessage( + ConfigNodeMessages + .MESSAGE_FAILED_ALTER_DATABASE_DOESN_T_SUPPORT_ALTER_TIMEPARTITIONORIGIN_YET_B315F2E3); + } + if (alterSchema.isSetTimePartitionInterval()) { + return new TSStatus(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode()) + .setMessage( + ConfigNodeMessages + .MESSAGE_FAILED_ALTER_DATABASE_DOESN_T_SUPPORT_ALTER_TIMEPARTITIONINTERVAL_YET_F539A76F); + } final PartialPath partialPathName = PartialPath.getQualifiedDatabasePartialPath(alterSchema.getName()); @@ -311,6 +326,9 @@ public TSStatus deleteDatabase(final DeleteDatabasePlan plan) { (isTableModel ? tableModelMTree : treeModelMTree) .deleteDatabase(getQualifiedDatabasePartialPath(plan.getName())); + // Remove database-specific time partition configuration from cache + TimePartitionUtils.removeDatabaseTimePartitionConfig(plan.getName()); + if (isTableModel) { configSchemaStatistics.decreaseTableDatabaseNum(); configSchemaStatistics.removeTableStatistics(plan.getName()); @@ -469,6 +487,7 @@ public TSStatus setDataReplicationFactor(final SetDataReplicationFactorPlan plan return result; } + /** Applies legacy consensus logs. New requests cannot reach this method. */ public TSStatus setTimePartitionInterval(final SetTimePartitionIntervalPlan plan) { final TSStatus result = new TSStatus(); databaseReadWriteLock.writeLock().lock(); @@ -477,11 +496,10 @@ public TSStatus setTimePartitionInterval(final SetTimePartitionIntervalPlan plan PathUtils.isTableModelDatabase(plan.getDatabase()) ? tableModelMTree : treeModelMTree; final PartialPath path = getQualifiedDatabasePartialPath(plan.getDatabase()); if (mTree.isDatabaseAlreadySet(path)) { - mTree - .getDatabaseNodeByDatabasePath(path) - .getAsMNode() - .getDatabaseSchema() - .setTimePartitionInterval(plan.getTimePartitionInterval()); + final TDatabaseSchema databaseSchema = + mTree.getDatabaseNodeByDatabasePath(path).getAsMNode().getDatabaseSchema(); + databaseSchema.setTimePartitionInterval(plan.getTimePartitionInterval()); + TimePartitionUtils.updateDatabaseTimePartitionConfig(plan.getDatabase(), databaseSchema); result.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } else { result.setCode(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode()); @@ -824,6 +842,7 @@ public void processLoadSnapshot(final File snapshotDir) throws IOException { }); templateTable.processLoadSnapshot(snapshotDir); templatePreSetTable.processLoadSnapshot(snapshotDir); + rebuildTimePartitionUtilsCache(); } public void processMTreeLoadSnapshot( @@ -853,6 +872,28 @@ public interface SerDeFunction { void apply(final T stream) throws IOException; } + private void rebuildTimePartitionUtilsCache() { + databaseReadWriteLock.writeLock().lock(); + try { + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); + syncTimePartitionUtilsCache(treeModelMTree); + syncTimePartitionUtilsCache(tableModelMTree); + } catch (final MetadataException e) { + LOGGER.warn("Failed to rebuild time partition cache from database schema.", e); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + + private void syncTimePartitionUtilsCache(final ConfigMTree mTree) throws MetadataException { + for (final PartialPath databasePath : mTree.getAllDatabasePaths(null)) { + final TDatabaseSchema databaseSchema = + mTree.getDatabaseNodeByPath(databasePath).getAsMNode().getDatabaseSchema(); + TimePartitionUtils.updateDatabaseTimePartitionConfig( + databaseSchema.getName(), databaseSchema); + } + } + public Pair, Set> getNodesListInGivenLevel( PartialPath partialPath, int level, PathPatternTree scope) { Pair, Set> matchedPathsInNextLevel = @@ -1684,6 +1725,7 @@ public ConfigSchemaStatistics getConfigSchemaStatistics() { public void clear() { treeModelMTree.clear(); tableModelMTree.clear(); + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); configSchemaStatistics.clear(); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedure.java index 0fc0f5d5b9d26..715387fe20196 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedure.java @@ -408,7 +408,7 @@ private Flow analyzeMissingPartitions(final ConfigNodeProcedureEnv env) { } if (localEarliestSlotStartTime - > TimePartitionUtils.getStartTimeByPartitionId(earliestTimeslot)) { + > TimePartitionUtils.getStartTimeByPartitionId(earliestTimeslot, database)) { databasesWithLostDataPartition.add(database); LOG.warn( ProcedureMessages diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 1f89e39d44883..da97549e22343 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -71,7 +71,6 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetDataReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetSchemaReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; -import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.consensus.response.ainode.AINodeConfigurationResp; import org.apache.iotdb.confignode.consensus.response.ainode.AINodeRegisterResp; @@ -201,7 +200,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TSetDataReplicationFactorReq; import org.apache.iotdb.confignode.rpc.thrift.TSetSchemaReplicationFactorReq; import org.apache.iotdb.confignode.rpc.thrift.TSetSchemaTemplateReq; -import org.apache.iotdb.confignode.rpc.thrift.TSetTimePartitionIntervalReq; import org.apache.iotdb.confignode.rpc.thrift.TShowAINodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowCQResp; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; @@ -531,13 +529,6 @@ public TSStatus setDataReplicationFactor(final TSetDataReplicationFactorReq req) new SetDataReplicationFactorPlan(req.getDatabase(), req.getDataReplicationFactor())); } - @Override - public TSStatus setTimePartitionInterval(final TSetTimePartitionIntervalReq req) - throws TException { - return configManager.setTimePartitionInterval( - new SetTimePartitionIntervalPlan(req.getDatabase(), req.getTimePartitionInterval())); - } - @Override public TCountDatabaseResp countMatchedDatabases(final TGetDatabaseReq req) { final PathPatternTree scope = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 3ec2538c6d44e..8e3878d1506fc 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -557,7 +557,8 @@ public void AutoCleanPartitionTablePlan() throws IOException { databaseTTLMap.put("root.db2", 3600L * 1000 * 24); // 1d_TTL databaseTTLMap.put("root.db3", 3600L * 1000 * 24 * 30); // 1m_TTL TTimePartitionSlot currentTimeSlot = - new TTimePartitionSlot(TimePartitionUtils.getTimePartitionSlot(System.currentTimeMillis())); + new TTimePartitionSlot( + TimePartitionUtils.getTimePartitionSlot(System.currentTimeMillis(), "")); AutoCleanPartitionTablePlan req0 = new AutoCleanPartitionTablePlan(databaseTTLMap, currentTimeSlot); AutoCleanPartitionTablePlan req1 = @@ -2292,10 +2293,21 @@ public void removeRegionLocationPlanTest() throws IOException { @Test public void GetRegionGroupsByTimePlanTest() throws IOException { - GetRegionGroupsByTimePlan plan0 = new GetRegionGroupsByTimePlan("root.sg0", 0L, 604800000L); - GetRegionGroupsByTimePlan plan1 = - (GetRegionGroupsByTimePlan) - ConfigPhysicalPlan.Factory.create(plan0.serializeToByteBuffer()); - Assert.assertEquals(plan0, plan1); + final String database = "root.sg0"; + final TDatabaseSchema databaseSchema = + new TDatabaseSchema(database).setTimePartitionOrigin(100L).setTimePartitionInterval(10L); + try { + TimePartitionUtils.updateDatabaseTimePartitionConfig(database, databaseSchema); + + GetRegionGroupsByTimePlan plan0 = new GetRegionGroupsByTimePlan(database, 105L, 125L); + Assert.assertEquals(100L, plan0.getStartTimeSlot().getStartTime()); + Assert.assertEquals(120L, plan0.getEndTimeSlot().getStartTime()); + GetRegionGroupsByTimePlan plan1 = + (GetRegionGroupsByTimePlan) + ConfigPhysicalPlan.Factory.create(plan0.serializeToByteBuffer()); + Assert.assertEquals(plan0, plan1); + } finally { + TimePartitionUtils.removeDatabaseTimePartitionConfig(database); + } } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java index 1f636e1d448da..8d3fbebf309f3 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java @@ -355,7 +355,7 @@ public void databaseWithoutTemplateGeneratorTest() throws Exception { tDatabaseSchema.setTTL(i + 1); tDatabaseSchema.setDataReplicationFactor(i); tDatabaseSchema.setSchemaReplicationFactor(i); - tDatabaseSchema.setTimePartitionInterval(i); + tDatabaseSchema.setTimePartitionInterval(i + 1); DatabaseSchemaPlan databaseSchemaPlan = new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, tDatabaseSchema); clusterSchemaInfo.createDatabase(databaseSchemaPlan); @@ -473,7 +473,7 @@ public void templateAndDatabaseCompletedTest() throws Exception { tDatabaseSchema.setTTL(i + 1); tDatabaseSchema.setDataReplicationFactor(i); tDatabaseSchema.setSchemaReplicationFactor(i); - tDatabaseSchema.setTimePartitionInterval(i); + tDatabaseSchema.setTimePartitionInterval(i + 1); final DatabaseSchemaPlan databaseSchemaPlan = new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, tDatabaseSchema); clusterSchemaInfo.createDatabase(databaseSchemaPlan); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfoTest.java index 82e94d083a40b..60989a7e68da8 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfoTest.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.table.ShowTablePlan; @@ -89,6 +90,7 @@ public class ClusterSchemaInfoTest { @Before public void setup() throws IOException { + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); clusterSchemaInfo = new ClusterSchemaInfo(); if (!snapshotDir.exists()) { snapshotDir.mkdirs(); @@ -98,6 +100,7 @@ public void setup() throws IOException { @After public void cleanup() throws IOException { clusterSchemaInfo.clear(); + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); if (snapshotDir.exists()) { FileUtils.deleteDirectory(snapshotDir); } @@ -112,12 +115,13 @@ public void testSnapshot() throws IOException, IllegalPathException { storageGroupPathList.add("root.a.a.a.b.sg"); Map testMap = new TreeMap<>(); - int i = 0; + int i = 1; for (String path : storageGroupPathList) { TDatabaseSchema tDatabaseSchema = new TDatabaseSchema(); tDatabaseSchema.setName(path); tDatabaseSchema.setDataReplicationFactor(i); tDatabaseSchema.setSchemaReplicationFactor(i); + tDatabaseSchema.setTimePartitionOrigin(i * 100L); tDatabaseSchema.setTimePartitionInterval(i); testMap.put(path, tDatabaseSchema); clusterSchemaInfo.createDatabase( @@ -148,6 +152,61 @@ public void testSnapshot() throws IOException, IllegalPathException { Map reloadResult = clusterSchemaInfo.getMatchedDatabaseSchemas(getStorageGroupReq).getSchemaMap(); Assert.assertEquals(testMap, reloadResult); + testMap.forEach( + (database, schema) -> { + Assert.assertEquals( + schema.getTimePartitionOrigin(), TimePartitionUtils.getTimePartitionOrigin(database)); + Assert.assertEquals( + schema.getTimePartitionInterval(), + TimePartitionUtils.getTimePartitionInterval(database)); + }); + } + + @Test + public void testTimePartitionConfigCacheConsistency() { + final TDatabaseSchema databaseSchema = + new TDatabaseSchema("root.sg").setTimePartitionOrigin(100L).setTimePartitionInterval(200L); + clusterSchemaInfo.createDatabase( + new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, databaseSchema)); + + Assert.assertEquals(100L, TimePartitionUtils.getTimePartitionOrigin("root.sg")); + Assert.assertEquals(200L, TimePartitionUtils.getTimePartitionInterval("root.sg")); + } + + @Test + public void testAlterDatabaseRejectsTimePartitionConfig() throws IllegalPathException { + final String database = "root.alter"; + final TDatabaseSchema databaseSchema = + new TDatabaseSchema(database) + .setIsTableModel(false) + .setTimePartitionOrigin(100L) + .setTimePartitionInterval(200L); + clusterSchemaInfo.createDatabase( + new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, databaseSchema)); + + final TSStatus intervalStatus = + clusterSchemaInfo.alterDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.AlterDatabase, + new TDatabaseSchema(database) + .setIsTableModel(false) + .setTimePartitionInterval(400L))); + Assert.assertEquals( + TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode(), intervalStatus.getCode()); + Assert.assertEquals(100L, TimePartitionUtils.getTimePartitionOrigin(database)); + Assert.assertEquals(200L, TimePartitionUtils.getTimePartitionInterval(database)); + + final GetDatabasePlan getDatabasePlan = + new GetDatabasePlan( + Arrays.asList(PathUtils.splitPathToDetachedNodes(database)), + ALL_MATCH_SCOPE, + false, + false, + false); + final TDatabaseSchema storedSchema = + clusterSchemaInfo.getMatchedDatabaseSchemas(getDatabasePlan).getSchemaMap().get(database); + Assert.assertEquals(100L, storedSchema.getTimePartitionOrigin()); + Assert.assertEquals(200L, storedSchema.getTimePartitionInterval()); } @Test diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java index 336bba31c3788..741a626aa1009 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java @@ -32,6 +32,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterResp; import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartReq; import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartResp; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TRuntimeConfiguration; import org.apache.iotdb.confignode.service.ConfigNode; import org.apache.iotdb.consensus.IConsensus; @@ -208,4 +209,24 @@ public void testDeleteConfigNodePeerShouldRejectMismatchedTarget() throws Except Assert.assertEquals(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode(), status.getCode()); Mockito.verify(configManager, Mockito.never()).getConsensusManager(); } + + public void testAlterDatabaseRejectsTimePartitionProperties() { + CommonConfig commonConfig = Mockito.mock(CommonConfig.class); + ConfigNodeConfig configNodeConfig = Mockito.mock(ConfigNodeConfig.class); + ConfigNode configNode = Mockito.mock(ConfigNode.class); + ConfigManager configManager = Mockito.mock(ConfigManager.class); + ConfigNodeRPCServiceProcessor sut = + new ConfigNodeRPCServiceProcessor( + commonConfig, configNodeConfig, configNode, configManager); + + TSStatus status = + sut.alterDatabase( + new TDatabaseSchema("root.sg") + .setIsTableModel(false) + .setTimePartitionOrigin(100L) + .setTimePartitionInterval(200L)); + + Assert.assertEquals(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode(), status.getCode()); + Mockito.verify(configManager, Mockito.never()).alterDatabase(Mockito.any()); + } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index db0fa44dbc590..34d4d356b7d30 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -2022,8 +2022,12 @@ public final class DataNodeQueryMessages { "' is currently not allowed."; public static final String VALUE_MUST_BE_A_LONGLITERAL_BUT_NOW_IS = " value must be a LongLiteral, but now is "; + public static final String EXCEPTION_ARG_VALUE_MUST_BE_A_BOOLEANLITERAL_BUT_NOW_IS_ARG_VALUE_ARG_25C5FFF0 = + "%s value must be a BooleanLiteral, but now is %s, value: %s"; public static final String VALUE = ", value: "; + public static final String LOG_FAILED_TO_GET_NEED_LAST_CACHE_INFO_FOR_DATABASE_ARG_WILL_PUT_CACHE_ANYWAY_EXCEPTION_ARG_D5D80E96 = + "Failed to get need_last_cache info for database {}, will put cache anyway, exception: {}"; public static final String VALUE_MUST_BE_EQUAL_TO_OR_GREATER_THAN_0_BUT_NOW_IS = " value must be equal to or greater than 0, but now is: "; public static final String VALUE_MUST_BE_LOWER_THAN = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index f500838c1ca2f..c93122773b331 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -2131,9 +2131,15 @@ public final class DataNodeQueryMessages { public static final String VALUE_MUST_BE_A_LONGLITERAL_BUT_NOW_IS = " 值必须为 LongLiteral，当前为 "; + public static final String EXCEPTION_ARG_VALUE_MUST_BE_A_BOOLEANLITERAL_BUT_NOW_IS_ARG_VALUE_ARG_25C5FFF0 = + + "%s 值必须为 BooleanLiteral，当前为 %s，值：%s"; public static final String VALUE = "，值："; + public static final String LOG_FAILED_TO_GET_NEED_LAST_CACHE_INFO_FOR_DATABASE_ARG_WILL_PUT_CACHE_ANYWAY_EXCEPTION_ARG_D5D80E96 = + + "获取数据库 {} 的 need_last_cache 信息失败，将继续写入缓存，异常：{}"; public static final String VALUE_MUST_BE_EQUAL_TO_OR_GREATER_THAN_0_BUT_NOW_IS = " 值必须大于等于 0，当前为："; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/partition/DataPartitionTableGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/partition/DataPartitionTableGenerator.java index 2f84fcbedbf7c..8bd7a453f8a82 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/partition/DataPartitionTableGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/partition/DataPartitionTableGenerator.java @@ -161,14 +161,16 @@ private void generateDataPartitionTableByMemory() { new ConcurrentHashMap<>(); tsFileManager.readLock(); - List seqTsFileList = tsFileManager.getTsFileList(true); - List unseqTsFileList = tsFileManager.getTsFileList(false); + List seqTsFileList = + tsFileManager.getTsFileList(true, databaseName); + List unseqTsFileList = + tsFileManager.getTsFileList(false, databaseName); tsFileManager.readUnlock(); constructDataPartitionMap( - seqTsFileList, seriesPartitionExecutor, dataPartitionMap); + seqTsFileList, seriesPartitionExecutor, dataPartitionMap, databaseName); constructDataPartitionMap( - unseqTsFileList, seriesPartitionExecutor, dataPartitionMap); + unseqTsFileList, seriesPartitionExecutor, dataPartitionMap, databaseName); if (dataPartitionMap.isEmpty()) { LOG.error(DataNodeMiscMessages.FAILED_GENERATE_DATA_PARTITION_TABLE); @@ -220,7 +222,8 @@ private void generateDataPartitionTableByMemory() { private void constructDataPartitionMap( List seqTsFileList, SeriesPartitionExecutor seriesPartitionExecutor, - Map dataPartitionMap) { + Map dataPartitionMap, + String database) { Set timeSlotIds = Collections.newSetFromMap(new ConcurrentHashMap<>()); for (TsFileResource tsFileResource : seqTsFileList) { @@ -237,10 +240,12 @@ private void constructDataPartitionMap( TSeriesPartitionSlot seriesSlotId = seriesPartitionExecutor.getSeriesPartitionSlot(deviceId); TTimePartitionSlot timePartitionSlot = - new TTimePartitionSlot(TimePartitionUtils.getStartTimeByPartitionId(timeSlotId)); + new TTimePartitionSlot( + TimePartitionUtils.getStartTimeByPartitionId(timeSlotId, database)); dataPartitionMap .computeIfAbsent( - seriesSlotId, empty -> newSeriesPartitionTable(consensusGroupId, timeSlotId)) + seriesSlotId, + empty -> newSeriesPartitionTable(consensusGroupId, timeSlotId, database)) .putDataPartition(timePartitionSlot, consensusGroupId); } if (!timeSlotIds.contains(timeSlotId)) { @@ -263,10 +268,10 @@ private void constructDataPartitionMap( } private static SeriesPartitionTable newSeriesPartitionTable( - TConsensusGroupId consensusGroupId, long timeSlotId) { + TConsensusGroupId consensusGroupId, long timeSlotId, String database) { SeriesPartitionTable seriesPartitionTable = new SeriesPartitionTable(); TTimePartitionSlot timePartitionSlot = - new TTimePartitionSlot(TimePartitionUtils.getStartTimeByPartitionId(timeSlotId)); + new TTimePartitionSlot(TimePartitionUtils.getStartTimeByPartitionId(timeSlotId, database)); seriesPartitionTable.putDataPartition(timePartitionSlot, consensusGroupId); return seriesPartitionTable; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ReplicateProgressDataNodeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ReplicateProgressDataNodeManager.java index 653fbe5c81bb8..c7b05d6f08c9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ReplicateProgressDataNodeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ReplicateProgressDataNodeManager.java @@ -102,13 +102,15 @@ private void recoverMaxProgressIndexFromDataRegion() { final TsFileManager tsFileManager = StorageEngine.getInstance().getDataRegion(dataRegionId).getTsFileManager(); + final String database = + StorageEngine.getInstance().getDataRegion(dataRegionId).getDatabaseName(); final List allProgressIndex = new ArrayList<>(); allProgressIndex.addAll( - tsFileManager.getTsFileList(true).stream() + tsFileManager.getTsFileList(true, database).stream() .map(TsFileResource::getMaxProgressIndex) .collect(Collectors.toList())); allProgressIndex.addAll( - tsFileManager.getTsFileList(false).stream() + tsFileManager.getTsFileList(false, database).stream() .map(TsFileResource::getMaxProgressIndex) .collect(Collectors.toList())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java index 041d33c2faffb..004d482fe7cc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java @@ -765,7 +765,7 @@ private void extractTsFiles( final HistoricalTsFileExtractionStatistics statistics = new HistoricalTsFileExtractionStatistics(); final Map> sequenceTsFileResources2TableNames = - tsFileManager.getTsFileList(true).stream() + tsFileManager.getTsFileList(true, dataRegion.getDatabaseName()).stream() .peek(originalResourceList::add) .filter(resource -> shouldExtractTsFileResource(resource, statistics)) .collect( @@ -780,7 +780,7 @@ private void extractTsFiles( filteredTsFileResources2TableNames.putAll(sequenceTsFileResources2TableNames); final Map> unSequenceTsFileResources2TableNames = - tsFileManager.getTsFileList(false).stream() + tsFileManager.getTsFileList(false, dataRegion.getDatabaseName()).stream() .peek(originalResourceList::add) .filter(resource -> shouldExtractTsFileResource(resource, statistics)) .collect( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java index 801051b991c75..3cf92d2bc7dec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java @@ -255,8 +255,9 @@ public void customize( final DataRegion dataRegion = StorageEngine.getInstance().getDataRegion(new DataRegionId(environment.getRegionId())); + final String databaseName; if (dataRegion != null) { - final String databaseName = dataRegion.getDatabaseName(); + databaseName = dataRegion.getDatabaseName(); if (databaseName != null) { if (PathUtils.isTableModelDatabase(databaseName)) { isDbNameCoveredByPattern = tablePattern.coversDb(databaseName); @@ -264,16 +265,14 @@ public void customize( isDbNameCoveredByPattern = treePattern.coversDb(databaseName); } } + } else { + databaseName = null; } startTimePartitionIdLowerBound = - TimePartitionUtils.isTimePartitionStartTime(realtimeDataExtractionStartTime) - ? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionStartTime) - : TimePartitionUtils.getTimePartitionId(realtimeDataExtractionStartTime) + 1; + getStartTimePartitionIdLowerBound(realtimeDataExtractionStartTime, databaseName); endTimePartitionIdUpperBound = - TimePartitionUtils.isTimePartitionStartTime(realtimeDataExtractionEndTime) - ? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) - : TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) - 1; + getEndTimePartitionIdUpperBound(realtimeDataExtractionEndTime, databaseName); final boolean isDoubleLiving = PipeSourceConstant.isDoubleLiving(parameters); if (isDoubleLiving) { @@ -692,6 +691,20 @@ public final String getTsFileParser() { return tsFileParser; } + static long getStartTimePartitionIdLowerBound(final long time, final String database) { + final long partitionId = TimePartitionUtils.getTimePartitionId(time, database); + return TimePartitionUtils.getTimePartitionSlot(time, database).getStartTime() == time + ? partitionId + : partitionId + 1; + } + + static long getEndTimePartitionIdUpperBound(final long time, final String database) { + final long partitionId = TimePartitionUtils.getTimePartitionId(time, database); + return TimePartitionUtils.getTimePartitionSlot(time, database).getStartTime() == time + ? partitionId + : partitionId - 1; + } + private void maySkipProgressIndexForRealtimeEvent(final PipeRealtimeEvent event) { if (PipeTsFileEpochProgressIndexKeeper.getInstance() .isProgressIndexAfterOrEquals( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 9f1e90fda10af..8b29a2c9475b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -155,7 +155,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TSetDataReplicationFactorReq; import org.apache.iotdb.confignode.rpc.thrift.TSetSchemaReplicationFactorReq; import org.apache.iotdb.confignode.rpc.thrift.TSetSchemaTemplateReq; -import org.apache.iotdb.confignode.rpc.thrift.TSetTimePartitionIntervalReq; import org.apache.iotdb.confignode.rpc.thrift.TShowAINodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowCQResp; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; @@ -448,12 +447,6 @@ public TSStatus setDataReplicationFactor(TSetDataReplicationFactorReq req) throw () -> client.setDataReplicationFactor(req), status -> !updateConfigNodeLeader(status)); } - @Override - public TSStatus setTimePartitionInterval(TSetTimePartitionIntervalReq req) throws TException { - return executeRemoteCallWithRetry( - () -> client.setTimePartitionInterval(req), status -> !updateConfigNodeLeader(status)); - } - @Override public TSchemaPartitionTableResp getSchemaPartitionTable(final TSchemaPartitionReq req) throws TException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 311bf04322ad4..4203d52c9decc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -981,6 +981,9 @@ public void setNeedUpdateScanNumForLastQuery(boolean needUpdateScanNumForLastQue } public Optional getDatabaseName() { + if (session == null) { + return Optional.empty(); + } return session.getDatabaseName(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index 11a69b1ffcab0..5bce4420cfec0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -368,12 +368,13 @@ protected void constructLine() { } columnBuilders[2].writeInt(currentDatabase.getSchemaReplicationFactor()); columnBuilders[3].writeInt(currentDatabase.getDataReplicationFactor()); - columnBuilders[4].writeLong(currentDatabase.getTimePartitionInterval()); - columnBuilders[5].writeInt(currentDatabase.getSchemaRegionNum()); - columnBuilders[6].writeInt(currentDatabase.getMaxSchemaRegionNum()); - columnBuilders[7].writeInt(currentDatabase.getDataRegionNum()); - columnBuilders[8].writeInt(currentDatabase.getMaxDataRegionNum()); - columnBuilders[9].writeBoolean(currentDatabase.isNeedLastCache()); + columnBuilders[4].writeLong(currentDatabase.getTimePartitionOrigin()); + columnBuilders[5].writeLong(currentDatabase.getTimePartitionInterval()); + columnBuilders[6].writeInt(currentDatabase.getSchemaRegionNum()); + columnBuilders[7].writeInt(currentDatabase.getMaxSchemaRegionNum()); + columnBuilders[8].writeInt(currentDatabase.getDataRegionNum()); + columnBuilders[9].writeInt(currentDatabase.getMaxDataRegionNum()); + columnBuilders[10].writeBoolean(currentDatabase.isNeedLastCache()); resultBuilder.declarePosition(); currentDatabase = null; } @@ -1691,7 +1692,10 @@ private Map getTablesToScan(DataRegion dataRegion, long timePartit private String getTableTypeName(final TTableInfo tableInfo) { if (tableInfo.isSetType()) { - return TableType.values()[tableInfo.getType()].getName(); + final int tableType = tableInfo.getType(); + if (tableType >= 0 && tableType < TableType.values().length) { + return TableType.values()[tableType].getName(); + } } return TableType.BASE_TABLE.getName(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index a1d5dc7a27f9e..673a3d77cb1eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -61,6 +61,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -195,45 +196,46 @@ public static String getDatabaseName( public static List computeTableDataPartitionParams( final InsertBaseStatement statement, final MPPQueryContext context) { + String database = getDatabaseName(statement, context); if (statement instanceof InsertTabletStatement) { final InsertTabletStatement insertTabletStatement = (InsertTabletStatement) statement; - final Map> timePartitionSlotMap = new HashMap<>(); + final Map> dataPartitionQueryParamMap = new HashMap<>(); for (int i = 0; i < insertTabletStatement.getRowCount(); i++) { - timePartitionSlotMap - .computeIfAbsent(insertTabletStatement.getTableDeviceID(i), id -> new HashSet<>()) - .add(insertTabletStatement.getTimePartitionSlot(i)); + dataPartitionQueryParamMap + .computeIfAbsent(insertTabletStatement.getTableDeviceID(i), id -> new ArrayList<>()) + .add(insertTabletStatement.getTimes()[i]); } - return computeDataPartitionParams(timePartitionSlotMap, getDatabaseName(statement, context)); + return computeDataPartitionParamsByTime(dataPartitionQueryParamMap, database); } else if (statement instanceof InsertMultiTabletsStatement) { final InsertMultiTabletsStatement insertMultiTabletsStatement = (InsertMultiTabletsStatement) statement; - final Map> timePartitionSlotMap = new HashMap<>(); + final Map> dataPartitionQueryParamMap = new HashMap<>(); for (final InsertTabletStatement insertTabletStatement : insertMultiTabletsStatement.getInsertTabletStatementList()) { for (int i = 0; i < insertTabletStatement.getRowCount(); i++) { - timePartitionSlotMap - .computeIfAbsent(insertTabletStatement.getTableDeviceID(i), id -> new HashSet<>()) - .add(insertTabletStatement.getTimePartitionSlot(i)); + dataPartitionQueryParamMap + .computeIfAbsent(insertTabletStatement.getTableDeviceID(i), id -> new ArrayList<>()) + .add(insertTabletStatement.getTimes()[i]); } } - return computeDataPartitionParams(timePartitionSlotMap, getDatabaseName(statement, context)); + return computeDataPartitionParamsByTime(dataPartitionQueryParamMap, database); } else if (statement instanceof InsertRowStatement) { final InsertRowStatement insertRowStatement = (InsertRowStatement) statement; - return computeDataPartitionParams( + return computeDataPartitionParamsByTime( Collections.singletonMap( insertRowStatement.getTableDeviceID(), - Collections.singleton(insertRowStatement.getTimePartitionSlot())), - getDatabaseName(statement, context)); + Collections.singletonList(insertRowStatement.getTime())), + database); } else if (statement instanceof InsertRowsStatement) { final InsertRowsStatement insertRowsStatement = (InsertRowsStatement) statement; - final Map> timePartitionSlotMap = new HashMap<>(); + final Map> dataPartitionQueryParamMap = new HashMap<>(); for (final InsertRowStatement insertRowStatement : insertRowsStatement.getInsertRowStatementList()) { - timePartitionSlotMap - .computeIfAbsent(insertRowStatement.getTableDeviceID(), id -> new HashSet<>()) - .add(insertRowStatement.getTimePartitionSlot()); + dataPartitionQueryParamMap + .computeIfAbsent(insertRowStatement.getTableDeviceID(), id -> new ArrayList<>()) + .add(insertRowStatement.getTime()); } - return computeDataPartitionParams(timePartitionSlotMap, getDatabaseName(statement, context)); + return computeDataPartitionParamsByTime(dataPartitionQueryParamMap, database); } throw new UnsupportedOperationException( DataNodeQueryMessages.COMPUTEDATAPARTITIONPARAMS_FOR + statement); @@ -241,48 +243,75 @@ public static List computeTableDataPartitionParams( public static List computeTreeDataPartitionParams( InsertBaseStatement statement, MPPQueryContext context) { + final String database = getDatabaseName(statement, context); if (statement instanceof InsertTabletStatement) { DataPartitionQueryParam dataPartitionQueryParam = - getTreeDataPartitionQueryParam((InsertTabletStatement) statement, context); + getTreeDataPartitionQueryParam((InsertTabletStatement) statement, database); return Collections.singletonList(dataPartitionQueryParam); + } else if (statement instanceof InsertRowStatement) { + InsertRowStatement insertRowStatement = (InsertRowStatement) statement; + return computeDataPartitionParamsByTime( + Collections.singletonMap( + insertRowStatement.getDevicePath().getIDeviceIDAsFullDevice(), + Collections.singletonList(insertRowStatement.getTime())), + database); } else if (statement instanceof InsertMultiTabletsStatement) { InsertMultiTabletsStatement insertMultiTabletsStatement = (InsertMultiTabletsStatement) statement; - Map> dataPartitionQueryParamMap = new HashMap<>(); + Map> dataPartitionQueryParamMap = new HashMap<>(); for (InsertTabletStatement insertTabletStatement : insertMultiTabletsStatement.getInsertTabletStatementList()) { - Set timePartitionSlotSet = + List timeList = dataPartitionQueryParamMap.computeIfAbsent( insertTabletStatement.getDevicePath().getIDeviceIDAsFullDevice(), - k -> new HashSet<>()); - timePartitionSlotSet.addAll(insertTabletStatement.getTimePartitionSlots()); + k -> new ArrayList<>()); + for (int i = 0; i < insertTabletStatement.getRowCount(); i++) { + timeList.add(insertTabletStatement.getTimes()[i]); + } + } + return computeDataPartitionParamsByTime(dataPartitionQueryParamMap, database); + } else if (statement instanceof InsertRowsOfOneDeviceStatement) { + final InsertRowsOfOneDeviceStatement insertRowsOfOneDeviceStatement = + (InsertRowsOfOneDeviceStatement) statement; + Map> dataPartitionQueryParamMap = new HashMap<>(); + List timeList = + dataPartitionQueryParamMap.computeIfAbsent( + insertRowsOfOneDeviceStatement.getDevicePath().getIDeviceIDAsFullDevice(), + k -> new ArrayList<>()); + for (InsertRowStatement insertRowStatement : + insertRowsOfOneDeviceStatement.getInsertRowStatementList()) { + timeList.add(insertRowStatement.getTime()); } - return computeDataPartitionParams( - dataPartitionQueryParamMap, getDatabaseName(statement, context)); + return computeDataPartitionParamsByTime(dataPartitionQueryParamMap, database); } else if (statement instanceof InsertRowsStatement) { final InsertRowsStatement insertRowsStatement = (InsertRowsStatement) statement; - Map> dataPartitionQueryParamMap = new HashMap<>(); + Map> dataPartitionQueryParamMap = new HashMap<>(); for (InsertRowStatement insertRowStatement : insertRowsStatement.getInsertRowStatementList()) { - Set timePartitionSlotSet = - dataPartitionQueryParamMap.computeIfAbsent( + dataPartitionQueryParamMap + .computeIfAbsent( insertRowStatement.getDevicePath().getIDeviceIDAsFullDevice(), - k -> new HashSet<>()); - timePartitionSlotSet.add(insertRowStatement.getTimePartitionSlot()); + k -> new ArrayList<>()) + .add(insertRowStatement.getTime()); } - return computeDataPartitionParams( - dataPartitionQueryParamMap, getDatabaseName(statement, context)); + return computeDataPartitionParamsByTime(dataPartitionQueryParamMap, database); } throw new UnsupportedOperationException( DataNodeQueryMessages.COMPUTEDATAPARTITIONPARAMS_FOR + statement); } private static DataPartitionQueryParam getTreeDataPartitionQueryParam( - InsertTabletStatement statement, MPPQueryContext context) { + InsertTabletStatement statement, String database) { DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID(statement.getDevicePath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(statement.getTimePartitionSlots()); - dataPartitionQueryParam.setDatabaseName(getDatabaseName(statement, context)); + List timeList = new ArrayList<>(); + for (int i = 0; i < statement.getRowCount(); i++) { + timeList.add(statement.getTimes()[i]); + } + dataPartitionQueryParam.setTimeList(timeList); + if (database != null) { + dataPartitionQueryParam.setDatabaseName(database); + } return dataPartitionQueryParam; } @@ -305,6 +334,26 @@ public static List computeDataPartitionParams( return dataPartitionQueryParams; } + private static List computeDataPartitionParamsByTime( + Map> dataPartitionQueryParamMap) { + return computeDataPartitionParamsByTime(dataPartitionQueryParamMap, null); + } + + private static List computeDataPartitionParamsByTime( + Map> dataPartitionQueryParamMap, String databaseName) { + List dataPartitionQueryParams = new ArrayList<>(); + for (Map.Entry> entry : dataPartitionQueryParamMap.entrySet()) { + DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); + dataPartitionQueryParam.setDeviceID(entry.getKey()); + dataPartitionQueryParam.setTimeList(entry.getValue()); + if (databaseName != null) { + dataPartitionQueryParam.setDatabaseName(databaseName); + } + dataPartitionQueryParams.add(dataPartitionQueryParam); + } + return dataPartitionQueryParams; + } + public static void validateSchema( IAnalysis analysis, InsertBaseStatement insertStatement, Runnable schemaValidation) { final long startTime = System.nanoTime(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 018f171b13252..8b08f24078ce1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2221,25 +2221,37 @@ private DataPartition fetchDataPartitionByDevices( Set deviceSet, ISchemaTree schemaTree, MPPQueryContext context) { long startTime = System.nanoTime(); try { - Pair, Pair> res = - getTimePartitionSlotList(context.getGlobalTimeFilter(), context); - // there is no satisfied time range - if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) { - return new DataPartition( - Collections.emptyMap(), - CONFIG.getSeriesPartitionExecutorClass(), - CONFIG.getSeriesPartitionSlotNum()); - } - Map> sgNameToQueryParamsMap = new HashMap<>(); + Map> databaseToDevices = new HashMap<>(); for (IDeviceID deviceID : deviceSet) { - DataPartitionQueryParam queryParam = - new DataPartitionQueryParam(deviceID, res.left, res.right.left, res.right.right); - sgNameToQueryParamsMap + databaseToDevices .computeIfAbsent(schemaTree.getBelongedDatabase(deviceID), key -> new ArrayList<>()) - .add(queryParam); + .add(deviceID); } + partitionFetcher.ensureDatabaseTimePartitionConfig(databaseToDevices.keySet()); - if (res.right.left || res.right.right) { + boolean hasUnclosedTimeRange = false; + Map> sgNameToQueryParamsMap = new HashMap<>(); + for (Map.Entry> entry : databaseToDevices.entrySet()) { + String database = entry.getKey(); + Pair, Pair> res = + getTimePartitionSlotList(context.getGlobalTimeFilter(), context, database); + // there is no satisfied time range + if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) { + return new DataPartition( + Collections.emptyMap(), + CONFIG.getSeriesPartitionExecutorClass(), + CONFIG.getSeriesPartitionSlotNum()); + } + hasUnclosedTimeRange |= res.right.left || res.right.right; + for (IDeviceID deviceID : entry.getValue()) { + sgNameToQueryParamsMap + .computeIfAbsent(database, key -> new ArrayList<>()) + .add( + new DataPartitionQueryParam(deviceID, res.left, res.right.left, res.right.right)); + } + } + + if (hasUnclosedTimeRange) { return partitionFetcher.getDataPartitionWithUnclosedTimeRange(sgNameToQueryParamsMap); } else { return partitionFetcher.getDataPartition(sgNameToQueryParamsMap); @@ -2252,6 +2264,11 @@ private DataPartition fetchDataPartitionByDevices( } } + public static Pair, Pair> getTimePartitionSlotList( + Filter timeFilter, MPPQueryContext context) { + return getTimePartitionSlotList(timeFilter, context, context.getDatabase()); + } + /** * get TTimePartitionSlot list about this time filter * @@ -2260,7 +2277,7 @@ private DataPartition fetchDataPartitionByDevices( */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning public static Pair, Pair> getTimePartitionSlotList( - Filter timeFilter, MPPQueryContext context) { + Filter timeFilter, MPPQueryContext context, String database) { if (timeFilter == null) { // (-oo, +oo) return new Pair<>(Collections.emptyList(), new Pair<>(true, true)); @@ -2285,11 +2302,15 @@ public static Pair, Pair> getTimePart if (timeRangeList.get(0).getMin() == Long.MIN_VALUE) { needLeftAll = true; - endTime = TimePartitionUtils.getTimePartitionUpperBound(timeRangeList.get(0).getMax()); - timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(timeRangeList.get(0).getMax()); + endTime = + TimePartitionUtils.getTimePartitionUpperBound(timeRangeList.get(0).getMax(), database); + timePartitionSlot = + TimePartitionUtils.getTimePartitionSlot(timeRangeList.get(0).getMax(), database); } else { - endTime = TimePartitionUtils.getTimePartitionUpperBound(timeRangeList.get(0).getMin()); - timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(timeRangeList.get(0).getMin()); + endTime = + TimePartitionUtils.getTimePartitionUpperBound(timeRangeList.get(0).getMin(), database); + timePartitionSlot = + TimePartitionUtils.getTimePartitionSlot(timeRangeList.get(0).getMin(), database); needLeftAll = false; } @@ -2303,26 +2324,26 @@ public static Pair, Pair> getTimePart List result = new ArrayList<>(); TimeRange currentTimeRange = timeRangeList.get(index); reserveMemoryForTimePartitionSlot( - currentTimeRange.getMax(), currentTimeRange.getMin(), context); + currentTimeRange.getMax(), currentTimeRange.getMin(), context, database); while (index < size) { long curLeft = timeRangeList.get(index).getMin(); long curRight = timeRangeList.get(index).getMax(); if (curLeft >= endTime) { result.add(timePartitionSlot); // next init - endTime = TimePartitionUtils.getTimePartitionUpperBound(curLeft); - timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(curLeft); + endTime = TimePartitionUtils.getTimePartitionUpperBound(curLeft, database); + timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(curLeft, database); } else if (curRight >= endTime) { result.add(timePartitionSlot); // next init timePartitionSlot = new TTimePartitionSlot(endTime); - endTime = TimePartitionUtils.getTimePartitionUpperBound(endTime); + endTime = TimePartitionUtils.getTimePartitionUpperBound(endTime, database); } else { index++; if (index < size) { currentTimeRange = timeRangeList.get(index); reserveMemoryForTimePartitionSlot( - currentTimeRange.getMax(), currentTimeRange.getMin(), context); + currentTimeRange.getMax(), currentTimeRange.getMin(), context, database); } } } @@ -2331,7 +2352,7 @@ public static Pair, Pair> getTimePart if (needRightAll) { TTimePartitionSlot lastTimePartitionSlot = TimePartitionUtils.getTimePartitionSlot( - timeRangeList.get(timeRangeList.size() - 1).getMin()); + timeRangeList.get(timeRangeList.size() - 1).getMin(), database); if (lastTimePartitionSlot.startTime != timePartitionSlot.startTime) { result.add(lastTimePartitionSlot); } @@ -2340,11 +2361,11 @@ public static Pair, Pair> getTimePart } private static void reserveMemoryForTimePartitionSlot( - long maxTime, long minTime, MPPQueryContext context) { + long maxTime, long minTime, MPPQueryContext context, String database) { if (maxTime == Long.MAX_VALUE || minTime == Long.MIN_VALUE) { return; } - long size = TimePartitionUtils.getEstimateTimePartitionSize(minTime, maxTime); + long size = TimePartitionUtils.getEstimateTimePartitionSize(minTime, maxTime, database); context.reserveMemoryForFrontEnd(estimateTimePartitionSlotMemory(size)); } @@ -2897,16 +2918,9 @@ public Analysis visitInsertRow(InsertRowStatement insertRowStatement, MPPQueryCo analysis.setRealStatement(realInsertStatement); if (realInsertStatement instanceof InsertRowStatement) { - InsertRowStatement realInsertRowStatement = (InsertRowStatement) realInsertStatement; - DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); - dataPartitionQueryParam.setDeviceID( - realInsertRowStatement.getDevicePath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList( - Collections.singletonList(realInsertRowStatement.getTimePartitionSlot())); - AnalyzeUtils.analyzeDataPartition( analysis, - Collections.singletonList(dataPartitionQueryParam), + AnalyzeUtils.computeTreeDataPartitionParams(realInsertStatement, context), context.getSession().getUserName(), partitionFetcher::getOrCreateDataPartition); } else { @@ -2987,15 +3001,9 @@ public Analysis visitInsertRowsOfOneDevice( analysis.setRealStatement(realInsertStatement); if (realInsertStatement instanceof InsertRowsOfOneDeviceStatement) { - InsertRowsOfOneDeviceStatement realStatement = - (InsertRowsOfOneDeviceStatement) realInsertStatement; - DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); - dataPartitionQueryParam.setDeviceID(realStatement.getDevicePath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(realStatement.getTimePartitionSlots()); - AnalyzeUtils.analyzeDataPartition( analysis, - Collections.singletonList(dataPartitionQueryParam), + AnalyzeUtils.computeTreeDataPartitionParams(realInsertStatement, context), context.getSession().getUserName(), partitionFetcher::getOrCreateDataPartition); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java index 824e3a4fbe08d..ac6fe25f1e5b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java @@ -33,8 +33,13 @@ import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchemaResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; import org.apache.iotdb.confignode.rpc.thrift.TSchemaNodeManagementReq; import org.apache.iotdb.confignode.rpc.thrift.TSchemaNodeManagementResp; import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionReq; @@ -61,6 +66,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -370,6 +376,49 @@ public void invalidReplicaSetCache(final TConsensusGroupId regionId) { partitionCache.invalidReplicaSetCache(regionId); } + @Override + public void ensureDatabaseTimePartitionConfig(Set databases) { + if (databases == null) { + return; + } + final Set missingDatabases = + databases.stream() + .filter(Objects::nonNull) + .filter(database -> !TimePartitionUtils.hasDatabaseTimePartitionConfig(database)) + .collect(Collectors.toSet()); + if (missingDatabases.isEmpty()) { + return; + } + final boolean needFetchTableModelDatabase = + missingDatabases.stream().anyMatch(PathUtils::isTableModelDatabase); + final boolean needFetchTreeModelDatabase = + missingDatabases.stream().anyMatch(database -> !PathUtils.isTableModelDatabase(database)); + try (final ConfigNodeClient client = + configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + if (needFetchTreeModelDatabase) { + updateDatabaseCache(client, false); + } + if (needFetchTableModelDatabase) { + updateDatabaseCache(client, true); + } + } catch (final ClientManagerException | TException e) { + throw new StatementAnalyzeException( + "An error occurred when fetching database time partition config:" + e.getMessage()); + } + } + + private void updateDatabaseCache(final ConfigNodeClient client, final boolean isTableModel) + throws TException { + final TDatabaseSchemaResp databaseSchemaResp = + client.getMatchedDatabaseSchemas( + new TGetDatabaseReq(Arrays.asList("root", "**"), SchemaConstant.ALL_MATCH_SCOPE_BINARY) + .setIsTableModel(isTableModel) + .setCanSeeAuditDB(true)); + if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + partitionCache.updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap()); + } + } + @Override public SchemaPartition getOrCreateSchemaPartition( final String database, final List deviceIDs, final String userName) { @@ -461,7 +510,22 @@ private Map> splitDataPartitionQueryParam( database = dataPartitionQueryParam.getDatabaseName(); } if (database != null) { - result.computeIfAbsent(database, key -> new ArrayList<>()).add(dataPartitionQueryParam); + if (PathUtils.isTableModelDatabase(database)) { + partitionCache.checkAndAutoCreateDatabase(database, isAutoCreate, userName); + } + ensureDatabaseTimePartitionConfig(Collections.singleton(database)); + final String finalDatabase = database; + if (dataPartitionQueryParam.hasTimeList()) { + dataPartitionQueryParam.setTimePartitionSlotList( + dataPartitionQueryParam.getTimeList().stream() + .map(time -> TimePartitionUtils.getTimePartitionSlot(time, finalDatabase)) + .distinct() + .collect(Collectors.toList())); + dataPartitionQueryParam.setDatabaseName(finalDatabase); + } + result + .computeIfAbsent(finalDatabase, key -> new ArrayList<>()) + .add(dataPartitionQueryParam); } } return result; @@ -550,11 +614,11 @@ private TDataPartitionReq constructDataPartitionReq( private TDataPartitionReq constructDataPartitionReqForQuery( final Map> sgNameToQueryParamsMap) { final Map> partitionSlotsMap = new HashMap<>(); - TTimeSlotList sharedTTimeSlotList = null; for (final Map.Entry> entry : sgNameToQueryParamsMap.entrySet()) { // for each sg final Map deviceToTimePartitionMap = new HashMap<>(); + TTimeSlotList sharedTTimeSlotList = null; for (final DataPartitionQueryParam queryParam : entry.getValue()) { if (sharedTTimeSlotList == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java index 6d6f59133af9c..8e1cc9874e055 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; public interface IPartitionFetcher { @@ -116,6 +117,10 @@ SchemaNodeManagementPartition getSchemaNodeManagementPartitionWithLevel( /** Invalid all partition cache */ void invalidAllCache(); + default void ensureDatabaseTimePartitionConfig(Set databases) { + // No-op for standalone and test implementations. + } + // ======================== Table Model Schema Partition Interface ======================== /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java index 350cc5d090ea5..dd4a4ad05402b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java @@ -405,25 +405,37 @@ private static DataPartition fetchDataPartitionByDevices( IPartitionFetcher partitionFetcher) { long startTime = System.nanoTime(); try { - Pair, Pair> res = - getTimePartitionSlotList(context.getGlobalTimeFilter(), context); - // there is no satisfied time range - if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) { - return new DataPartition( - Collections.emptyMap(), - CONFIG.getSeriesPartitionExecutorClass(), - CONFIG.getSeriesPartitionSlotNum()); - } - Map> sgNameToQueryParamsMap = new HashMap<>(); + Map> databaseToDevices = new HashMap<>(); for (IDeviceID deviceID : deviceSet) { - DataPartitionQueryParam queryParam = - new DataPartitionQueryParam(deviceID, res.left, res.right.left, res.right.right); - sgNameToQueryParamsMap + databaseToDevices .computeIfAbsent(schemaTree.getBelongedDatabase(deviceID), key -> new ArrayList<>()) - .add(queryParam); + .add(deviceID); + } + partitionFetcher.ensureDatabaseTimePartitionConfig(databaseToDevices.keySet()); + + boolean hasUnclosedTimeRange = false; + Map> sgNameToQueryParamsMap = new HashMap<>(); + for (Map.Entry> entry : databaseToDevices.entrySet()) { + String database = entry.getKey(); + Pair, Pair> res = + getTimePartitionSlotList(context.getGlobalTimeFilter(), context, database); + // there is no satisfied time range + if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) { + return new DataPartition( + Collections.emptyMap(), + CONFIG.getSeriesPartitionExecutorClass(), + CONFIG.getSeriesPartitionSlotNum()); + } + hasUnclosedTimeRange |= res.right.left || res.right.right; + for (IDeviceID deviceID : entry.getValue()) { + sgNameToQueryParamsMap + .computeIfAbsent(database, key -> new ArrayList<>()) + .add( + new DataPartitionQueryParam(deviceID, res.left, res.right.left, res.right.right)); + } } - if (res.right.left || res.right.right) { + if (hasUnclosedTimeRange) { return partitionFetcher.getDataPartitionWithUnclosedTimeRange(sgNameToQueryParamsMap); } else { return partitionFetcher.getDataPartition(sgNameToQueryParamsMap); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 829f14d615968..f6b8610627781 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -46,6 +46,7 @@ import org.apache.iotdb.commons.schema.table.Audit; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; @@ -226,12 +227,17 @@ public void put(final IDeviceID device, final String databaseName) { * @return database name, return {@code null} if cache miss */ private String getDatabaseName(final IDeviceID deviceID) { - for (final String database : database2NeedLastCacheCache.keySet()) { - if (PathUtils.isStartWith(deviceID, database)) { - return database; + databaseCacheLock.readLock().lock(); + try { + for (final String database : database2NeedLastCacheCache.keySet()) { + if (PathUtils.isStartWith(deviceID, database)) { + return database; + } } + return null; + } finally { + databaseCacheLock.readLock().unlock(); } - return null; } /** @@ -286,7 +292,13 @@ private void fetchDatabaseAndUpdateCache( @TreeModel public boolean isNeedLastCache(final String database) { - Boolean needLastCache = database2NeedLastCacheCache.get(database); + databaseCacheLock.readLock().lock(); + Boolean needLastCache; + try { + needLastCache = database2NeedLastCacheCache.get(database); + } finally { + databaseCacheLock.readLock().unlock(); + } if (Objects.nonNull(needLastCache)) { return needLastCache; } @@ -294,12 +306,18 @@ public boolean isNeedLastCache(final String database) { fetchDatabaseAndUpdateCache(false); } catch (final TException | ClientManagerException e) { logger.warn( - "Failed to get need_last_cache info for database {}, will put cache anyway, exception: {}", + DataNodeQueryMessages + .LOG_FAILED_TO_GET_NEED_LAST_CACHE_INFO_FOR_DATABASE_ARG_WILL_PUT_CACHE_ANYWAY_EXCEPTION_ARG_D5D80E96, database, e.getMessage()); return true; } - needLastCache = database2NeedLastCacheCache.get(database); + databaseCacheLock.readLock().lock(); + try { + needLastCache = database2NeedLastCacheCache.get(database); + } finally { + databaseCacheLock.readLock().unlock(); + } return Objects.isNull(needLastCache) || needLastCache; } @@ -372,6 +390,8 @@ private void createDatabaseAndUpdateCache( // Try to create databases one by one until done or one database fail final Set successFullyCreatedDatabase = new HashSet<>(); + final Map successFullyCreatedDatabaseSchema = new HashMap<>(); + boolean needRefreshCacheFromConfigNode = false; for (final String databaseName : databaseNamesNeedCreated) { final long startTime = System.nanoTime(); try { @@ -391,9 +411,17 @@ private void createDatabaseAndUpdateCache( databaseSchema.setName(databaseName); databaseSchema.setIsTableModel(false); final TSStatus tsStatus = client.setDatabase(databaseSchema); - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() - || TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() == tsStatus.getCode()) { + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode()) { + successFullyCreatedDatabase.add(databaseName); + successFullyCreatedDatabaseSchema.put(databaseName, databaseSchema); + // In tree model, if the user creates a conflict database concurrently, for instance, + // the database created by user is root.db.ss.a, the auto-creation failed database is + // root.db, we wait till "getOrCreatePartition" to judge if the time series (like + // root.db.ss.a.e / root.db.ss.a) conflicts with the created database. just do not throw + // exception here. + } else if (TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() == tsStatus.getCode()) { successFullyCreatedDatabase.add(databaseName); + needRefreshCacheFromConfigNode = true; // In tree model, if the user creates a conflict database concurrently, for instance, // the database created by user is root.db.ss.a, the auto-creation failed database is // root.db, we wait till "getOrCreatePartition" to judge if the time series (like @@ -401,7 +429,11 @@ private void createDatabaseAndUpdateCache( // exception here. } else if (TSStatusCode.DATABASE_CONFLICT.getStatusCode() != tsStatus.getCode()) { // Try to update cache by databases successfully created - updateDatabaseCache(successFullyCreatedDatabase); + if (needRefreshCacheFromConfigNode) { + fetchDatabaseAndUpdateCache(client, false); + } else { + updateDatabaseCache(successFullyCreatedDatabaseSchema); + } logger.warn( DataNodeQueryMessages.ARG_CACHE_FAILED_TO_CREATE_DATABASE_ARG, CacheMetrics.DATABASE_CACHE_NAME, @@ -409,8 +441,13 @@ private void createDatabaseAndUpdateCache( throw new IoTDBRuntimeException(tsStatus.message, tsStatus.code); } } - // Try to update database cache when all databases have already been created - updateDatabaseCache(successFullyCreatedDatabase); + // Fetch the completed schema from ConfigNode after auto-creation. The locally constructed + // schema may miss default time partition settings that ConfigNode fills in. + if (needRefreshCacheFromConfigNode || !successFullyCreatedDatabase.isEmpty()) { + fetchDatabaseAndUpdateCache(client, false); + } else { + updateDatabaseCache(successFullyCreatedDatabaseSchema); + } getDatabaseMap(result, deviceIDs, false); } } finally { @@ -445,10 +482,12 @@ private void createDatabaseAndUpdateCache(final String database, final String us databaseSchema.setName(database); databaseSchema.setIsTableModel(true); final TSStatus tsStatus = client.setDatabase(databaseSchema); - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() - || TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() == tsStatus.getCode()) { - // Try to update cache by databases successfully created - updateDatabaseCache(Collections.singleton(database)); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode()) { + // Fetch the completed schema from ConfigNode after auto-creation. The locally constructed + // schema may miss default time partition settings that ConfigNode fills in. + fetchDatabaseAndUpdateCache(client, true); + } else if (TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() == tsStatus.getCode()) { + fetchDatabaseAndUpdateCache(client, true); } else { logger.warn( DataNodeQueryMessages.ARG_CACHE_FAILED_TO_CREATE_DATABASE_ARG, @@ -637,16 +676,23 @@ public void updateDatabaseCache(final Set databases) { * @param databaseMap the database names and need last cache that need to update */ public void updateDatabaseCache(final Map databaseMap) { + if (databaseMap == null || databaseMap.isEmpty()) { + return; + } databaseCacheLock.writeLock().lock(); try { databaseMap.forEach( (database, schema) -> { + if (database == null || schema == null) { + return; + } final boolean needLastCache = isNeedLastCacheEnabled(schema); database2NeedLastCacheCache.put(database, needLastCache); if (!needLastCache) { TreeDeviceSchemaCacheManager.getInstance().invalidateDatabaseLastCache(database); } }); + TimePartitionUtils.updateDatabaseTimePartitionConfigs(databaseMap); } finally { databaseCacheLock.writeLock().unlock(); } @@ -657,11 +703,24 @@ public void removeFromDatabaseCache() { databaseCacheLock.writeLock().lock(); try { database2NeedLastCacheCache.clear(); + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); } finally { databaseCacheLock.writeLock().unlock(); } } + private void fetchDatabaseAndUpdateCache( + final ConfigNodeClient client, final boolean isTableModel) throws TException { + final TGetDatabaseReq req = + new TGetDatabaseReq(ROOT_PATH, SchemaConstant.ALL_MATCH_SCOPE_BINARY) + .setIsTableModel(isTableModel) + .setCanSeeAuditDB(true); + final TDatabaseSchemaResp databaseSchemaResp = client.getMatchedDatabaseSchemas(req); + if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap()); + } + } + // endregion // region replicaSet cache @@ -1274,9 +1333,16 @@ public void invalidAllCache() { @Override public String toString() { + final Map databaseCacheSnapshot; + databaseCacheLock.readLock().lock(); + try { + databaseCacheSnapshot = new HashMap<>(database2NeedLastCacheCache); + } finally { + databaseCacheLock.readLock().unlock(); + } return "PartitionCache{" + ", databaseCache=" - + database2NeedLastCacheCache + + databaseCacheSnapshot + ", replicaSetCache=" + groupIdToReplicaSetMap + ", schemaPartitionCache=" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index f91969ae32ad3..9f2e86eabc5e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -307,6 +307,7 @@ import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_DATA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_SCHEMA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.TIME_PARTITION_INTERVAL_KEY; +import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.TIME_PARTITION_ORIGIN_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.TTL_KEY; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; @@ -373,6 +374,7 @@ private IConfigTask visitDatabaseStatement( final String key = property.getName().getValue().toLowerCase(Locale.ENGLISH); if (property.isSetToDefault()) { switch (key) { + case TIME_PARTITION_ORIGIN_KEY: case TIME_PARTITION_INTERVAL_KEY: case MAX_SCHEMA_REGION_GROUP_NUM_KEY: case MAX_DATA_REGION_GROUP_NUM_KEY: @@ -411,6 +413,9 @@ private IConfigTask visitDatabaseStatement( } schema.setTTL(parseLongFromLiteral(value, TTL_KEY)); break; + case TIME_PARTITION_ORIGIN_KEY: + schema.setTimePartitionOrigin(parseLongFromLiteral(value, TIME_PARTITION_ORIGIN_KEY)); + break; case TIME_PARTITION_INTERVAL_KEY: schema.setTimePartitionInterval(parseLongFromLiteral(value, TIME_PARTITION_INTERVAL_KEY)); break; @@ -1231,11 +1236,12 @@ public IConfigTask visitSetSystemStatus(SetSystemStatus node, MPPQueryContext co private boolean parseBooleanFromLiteral(final Object value, final String name) { if (!(value instanceof BooleanLiteral)) { throw new SemanticException( - name - + " value must be a BooleanLiteral, but now is " - + (Objects.nonNull(value) ? value.getClass().getSimpleName() : null) - + ", value: " - + value); + String.format( + DataNodeQueryMessages + .EXCEPTION_ARG_VALUE_MUST_BE_A_BOOLEANLITERAL_BUT_NOW_IS_ARG_VALUE_ARG_25C5FFF0, + name, + Objects.nonNull(value) ? value.getClass().getSimpleName() : null, + value)); } return ((BooleanLiteral) value).getValue(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index a995d643970f4..283ab1a91cb9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -126,6 +126,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRemoveResp; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TDeactivateSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteDatabasesReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq; @@ -494,14 +495,42 @@ public SettableFuture setDatabase( } } else { + refreshDatabaseTimePartitionConfig(configNodeClient, databaseSchema); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); } - } catch (final ClientManagerException | TException e) { + } catch (final IOException | ClientManagerException | TException e) { future.setException(e); } return future; } + private void refreshDatabaseTimePartitionConfig( + final ConfigNodeClient configNodeClient, final TDatabaseSchema databaseSchema) + throws IOException, TException { + TDatabaseSchema completeDatabaseSchema = databaseSchema; + if (!databaseSchema.isSetTimePartitionOrigin() + || !databaseSchema.isSetTimePartitionInterval()) { + final TGetDatabaseReq req = + new TGetDatabaseReq( + Arrays.asList(ROOT, MULTI_LEVEL_PATH_WILDCARD), ALL_MATCH_SCOPE.serialize()) + .setIsTableModel( + databaseSchema.isSetIsTableModel() && databaseSchema.isIsTableModel()) + .setCanSeeAuditDB(true); + final TDatabaseSchemaResp resp = configNodeClient.getMatchedDatabaseSchemas(req); + if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + && resp.getDatabaseSchemaMap() != null) { + completeDatabaseSchema = resp.getDatabaseSchemaMap().get(databaseSchema.getName()); + } + } + + if (completeDatabaseSchema == null) { + TimePartitionUtils.removeDatabaseTimePartitionConfig(databaseSchema.getName()); + return; + } + TimePartitionUtils.updateDatabaseTimePartitionConfig( + completeDatabaseSchema.getName(), completeDatabaseSchema); + } + @Override public SettableFuture alterDatabase( final DatabaseSchemaStatement databaseSchemaStatement) { @@ -602,6 +631,7 @@ public SettableFuture deleteDatabase( future.setException(new IoTDBException(tsStatus)); } } else { + ClusterPartitionFetcher.getInstance().invalidAllCache(); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); } } catch (final ClientManagerException | TException e) { @@ -3797,9 +3827,11 @@ public SettableFuture getRegionId( tGetRegionIdReq.setDatabase(getRegionIdStatement.getDatabase()); } tGetRegionIdReq.setStartTimeSlot( - TimePartitionUtils.getTimePartitionSlot(getRegionIdStatement.getStartTimeStamp())); + TimePartitionUtils.getTimePartitionSlot( + getRegionIdStatement.getStartTimeStamp(), getRegionIdStatement.getDatabase())); tGetRegionIdReq.setEndTimeSlot( - TimePartitionUtils.getTimePartitionSlot(getRegionIdStatement.getEndTimeStamp())); + TimePartitionUtils.getTimePartitionSlot( + getRegionIdStatement.getEndTimeStamp(), getRegionIdStatement.getDatabase())); resp = configNodeClient.getRegionId(tGetRegionIdReq); if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(resp.getStatus())); @@ -3866,7 +3898,7 @@ public SettableFuture getTimeSlotList( } catch (final Exception e) { future.setException(e); } - GetTimeSlotListTask.buildTSBlock(resp, future); + GetTimeSlotListTask.buildTSBlock(resp, future, getTimeSlotListStatement.getDatabase()); return future; } @@ -4695,6 +4727,7 @@ public SettableFuture dropDatabase( CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TSStatus tsStatus = client.deleteDatabases(req); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode()) { + ClusterPartitionFetcher.getInstance().invalidAllCache(); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); unsetDatabaseIfNotExist(dropDB.getDbName().getValue(), session); } else if (TSStatusCode.PATH_NOT_EXIST.getStatusCode() == tsStatus.getCode()) { @@ -4746,6 +4779,7 @@ public SettableFuture createDatabase( final TSStatus tsStatus = configNodeClient.setDatabase(databaseSchema); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode()) { + refreshDatabaseTimePartitionConfig(configNodeClient, databaseSchema); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); } else if (TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() == tsStatus.getCode()) { if (ifExists) { @@ -4761,7 +4795,7 @@ public SettableFuture createDatabase( } else { future.setException(new IoTDBException(tsStatus)); } - } catch (final ClientManagerException | TException e) { + } catch (final IOException | ClientManagerException | TException e) { future.setException(e); } return future; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java index 581b1ed2b9980..e57e7240493ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java @@ -66,6 +66,9 @@ public static TDatabaseSchema constructDatabaseSchema( if (databaseSchemaStatement.getTimePartitionInterval() != null) { databaseSchema.setTimePartitionInterval(databaseSchemaStatement.getTimePartitionInterval()); } + if (databaseSchemaStatement.getTimePartitionOrigin() != null) { + databaseSchema.setTimePartitionOrigin(databaseSchemaStatement.getTimePartitionOrigin()); + } if (databaseSchemaStatement.getMaxSchemaRegionGroupNum() != null) { databaseSchema.setMaxSchemaRegionGroupNum( databaseSchemaStatement.getMaxSchemaRegionGroupNum()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/GetTimeSlotListTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/GetTimeSlotListTask.java index 2e0037e91e07a..b2e8f6c3f5eec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/GetTimeSlotListTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/GetTimeSlotListTask.java @@ -58,11 +58,13 @@ public ListenableFuture execute(IConfigTaskExecutor configTask return configTaskExecutor.getTimeSlotList(getTimeSlotListStatement); } - public static void buildTSBlockRow(TsBlockBuilder builder, TTimePartitionSlot timePartitionSlot) { + public static void buildTSBlockRow( + TsBlockBuilder builder, TTimePartitionSlot timePartitionSlot, String database) { builder.getTimeColumnBuilder().writeLong(0L); builder .getColumnBuilder(0) - .writeLong(TimePartitionUtils.getTimePartitionId(timePartitionSlot.getStartTime())); + .writeLong( + TimePartitionUtils.getTimePartitionId(timePartitionSlot.getStartTime(), database)); builder .getColumnBuilder(1) .writeBinary( @@ -73,14 +75,16 @@ public static void buildTSBlockRow(TsBlockBuilder builder, TTimePartitionSlot ti } public static void buildTSBlock( - TGetTimeSlotListResp getTimeSlotListResp, SettableFuture future) { + TGetTimeSlotListResp getTimeSlotListResp, + SettableFuture future, + String database) { List outputDataTypes = ColumnHeaderConstant.getTimeSlotListColumnHeaders.stream() .map(ColumnHeader::getColumnType) .collect(Collectors.toList()); TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); - getTimeSlotListResp.getTimeSlotList().forEach(e -> buildTSBlockRow(builder, e)); + getTimeSlotListResp.getTimeSlotList().forEach(e -> buildTSBlockRow(builder, e, database)); DatasetHeader datasetHeader = DatasetHeaderFactory.getGetTimeSlotListHeader(); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java index c1b6ba6bfc329..c7a53dd74d59f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java @@ -26,6 +26,7 @@ public abstract class AbstractDatabaseTask implements IConfigTask { /////////////////////////////// Allowed properties /////////////////////////////// public static final String TTL_KEY = "ttl"; + public static final String TIME_PARTITION_ORIGIN_KEY = "time_partition_origin"; public static final String TIME_PARTITION_INTERVAL_KEY = "time_partition_interval"; public static final String MAX_SCHEMA_REGION_GROUP_NUM_KEY = "max_schema_region_group_num"; public static final String MAX_DATA_REGION_GROUP_NUM_KEY = "max_data_region_group_num"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateDatabaseTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateDatabaseTask.java index 590d6fd30a01c..a759267c3860c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateDatabaseTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateDatabaseTask.java @@ -84,12 +84,16 @@ public static String getShowCreateDatabaseSQL(final TDatabaseInfo databaseInfo) databaseInfo.getTTL() == Long.MAX_VALUE ? ShowCreateTableTask.getString(IoTDBConstant.TTL_INFINITE) : databaseInfo.getTTL()) + .append(",time_partition_origin=") + .append(databaseInfo.getTimePartitionOrigin()) .append(",time_partition_interval=") .append(databaseInfo.getTimePartitionInterval()) .append(",max_schema_region_group_num=") .append(databaseInfo.getMaxSchemaRegionNum()) .append(",max_data_region_group_num=") .append(databaseInfo.getMaxDataRegionNum()) + .append(",need_last_cache=") + .append(databaseInfo.isSetNeedLastCache() ? databaseInfo.isNeedLastCache() : true) .append(")") .toString(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java index 05999e3f69dce..bd6ffb70b952b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java @@ -164,12 +164,13 @@ private static void buildTSBlockForDetails( } builder.getColumnBuilder(2).writeInt(storageGroupInfo.getSchemaReplicationFactor()); builder.getColumnBuilder(3).writeInt(storageGroupInfo.getDataReplicationFactor()); - builder.getColumnBuilder(4).writeLong(storageGroupInfo.getTimePartitionInterval()); - builder.getColumnBuilder(5).writeInt(storageGroupInfo.getSchemaRegionNum()); - builder.getColumnBuilder(6).writeInt(storageGroupInfo.getMaxSchemaRegionNum()); - builder.getColumnBuilder(7).writeInt(storageGroupInfo.getDataRegionNum()); - builder.getColumnBuilder(8).writeInt(storageGroupInfo.getMaxDataRegionNum()); - builder.getColumnBuilder(9).writeBoolean(storageGroupInfo.isNeedLastCache()); + builder.getColumnBuilder(4).writeLong(storageGroupInfo.getTimePartitionOrigin()); + builder.getColumnBuilder(5).writeLong(storageGroupInfo.getTimePartitionInterval()); + builder.getColumnBuilder(6).writeInt(storageGroupInfo.getSchemaRegionNum()); + builder.getColumnBuilder(7).writeInt(storageGroupInfo.getMaxSchemaRegionNum()); + builder.getColumnBuilder(8).writeInt(storageGroupInfo.getDataRegionNum()); + builder.getColumnBuilder(9).writeInt(storageGroupInfo.getMaxDataRegionNum()); + builder.getColumnBuilder(10).writeBoolean(storageGroupInfo.isNeedLastCache()); builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 50a4996c2ad88..4e1ce9aa74de3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -3032,6 +3032,9 @@ private void parseDatabaseAttributesClause( } else if (attributeKey.TIME_PARTITION_INTERVAL() != null) { final long timePartitionInterval = Long.parseLong(attribute.INTEGER_LITERAL().getText()); databaseSchemaStatement.setTimePartitionInterval(timePartitionInterval); + } else if (attributeKey.TIME_PARTITION_ORIGIN() != null) { + final long timePartitionOrigin = Long.parseLong(attribute.INTEGER_LITERAL().getText()); + databaseSchemaStatement.setTimePartitionOrigin(timePartitionOrigin); } else if (attributeKey.MAX_SCHEMA_REGION_GROUP_NUM() != null) { final int maxSchemaRegionGroupNum = Integer.parseInt(attribute.INTEGER_LITERAL().getText()); databaseSchemaStatement.setMaxSchemaRegionGroupNum(maxSchemaRegionGroupNum); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java index 2e3714b71a6c5..6b19abaae6165 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java @@ -1248,7 +1248,8 @@ private List splitInnerTimeJoinNode( InnerTimeJoinNode innerTimeJoinNode = (InnerTimeJoinNode) node.clone(); innerTimeJoinNode.setPlanNodeId(context.queryContext.getQueryId().genPlanNodeId()); - List timePartitionIds = convertToTimePartitionIds(oneRegion); + List timePartitionIds = + convertToTimePartitionIds(oneRegion, analysis.getDatabaseName()); innerTimeJoinNode.setTimePartitions(timePartitionIds); // region group id -> parent InnerTimeJoinNode @@ -1297,10 +1298,11 @@ private List splitInnerTimeJoinNode( return subInnerJoinNode; } - private List convertToTimePartitionIds(List timePartitionSlotList) { + private List convertToTimePartitionIds( + List timePartitionSlotList, String database) { List res = new ArrayList<>(timePartitionSlotList.size()); for (TTimePartitionSlot timePartitionSlot : timePartitionSlotList) { - res.add(TimePartitionUtils.getTimePartitionId(timePartitionSlot.startTime)); + res.add(TimePartitionUtils.getTimePartitionId(timePartitionSlot.startTime, database)); } return res; } @@ -1464,14 +1466,18 @@ private List getDeviceReplicaSets( tSeriesPartitionSlot.slotId, k -> getDataRegionReplicaSetWithTimeFilter( - finalSeriesPartitionMap, tSeriesPartitionSlot, timeFilter)); + finalSeriesPartitionMap, + tSeriesPartitionSlot, + timeFilter, + analysis.getDatabaseName())); } public List getDataRegionReplicaSetWithTimeFilter( Map>> seriesPartitionMap, TSeriesPartitionSlot tSeriesPartitionSlot, - Filter timeFilter) { + Filter timeFilter, + String database) { Map> regionReplicaSetMap = seriesPartitionMap.getOrDefault(tSeriesPartitionSlot, Collections.emptyMap()); if (regionReplicaSetMap.isEmpty()) { @@ -1481,7 +1487,8 @@ public List getDataRegionReplicaSetWithTimeFilter( Set uniqueValues = new HashSet<>(); for (Map.Entry> entry : regionReplicaSetMap.entrySet()) { - if (!TimePartitionUtils.satisfyPartitionStartTime(timeFilter, entry.getKey().startTime)) { + if (!TimePartitionUtils.satisfyPartitionStartTime( + timeFilter, entry.getKey().startTime, database)) { continue; } for (TRegionReplicaSet tRegionReplicaSet : entry.getValue()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index ecaf0aea6ec2f..743166422f7e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -102,9 +102,9 @@ public boolean needDecodeTsFile( for (final IDeviceID device : resource.getDevices()) { // iterating the index, must present final TTimePartitionSlot startSlot = - TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(device).get()); + TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(device).get(), database); final TTimePartitionSlot endSlot = - TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(device).get()); + TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(device).get(), database); slotList.add(new Pair<>(device, startSlot)); if (!startSlot.equals(endSlot)) { slotList.add(new Pair<>(device, endSlot)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index 0f2bc190120f2..54f2378e29b07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.runtime.SerializationRunTimeException; +import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; @@ -33,6 +34,7 @@ import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory; @@ -250,6 +252,16 @@ public IDeviceID getDeviceID() { return deviceID; } + protected String getDatabaseName(final IAnalysis analysis, final IDeviceID deviceID) { + final String databaseName = analysis.getDatabaseName(); + final DataPartition dataPartitionInfo = analysis.getDataPartitionInfo(); + final String databaseNameInPartition = + dataPartitionInfo == null || dataPartitionInfo.isEmpty() + ? null + : dataPartitionInfo.getDatabaseNameByDevice(deviceID); + return databaseNameInPartition != null ? databaseNameInPartition : databaseName; + } + public void setDeviceID(IDeviceID deviceID) { this.deviceID = deviceID; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index ecf76892e4cd9..e0270e968d60a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -45,6 +45,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotImplementedException; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -128,12 +129,14 @@ public InsertRowNode( @Override public List splitByPartition(IAnalysis analysis) { - TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(time); + final IDeviceID deviceID = getDeviceID(); + final String databaseName = getDatabaseName(analysis, deviceID); + TTimePartitionSlot timePartitionSlot = + TimePartitionUtils.getTimePartitionSlot(time, databaseName); this.dataRegionReplicaSet = analysis .getDataPartitionInfo() - .getDataRegionReplicaSetForWriting( - getDeviceID(), timePartitionSlot, analysis.getDatabaseName()); + .getDataRegionReplicaSetForWriting(deviceID, timePartitionSlot, databaseName); // collect redirectInfo analysis.setRedirectNodeList( Collections.singletonList( @@ -234,8 +237,8 @@ public void setNeedInferType(boolean needInferType) { } @TestOnly - public List getTimePartitionSlots() { - return Collections.singletonList(TimePartitionUtils.getTimePartitionSlot(time)); + public List getTimePartitionSlots(String database) { + return Collections.singletonList(TimePartitionUtils.getTimePartitionSlot(time, database)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java index c2ba7c04791ec..e2d3f5ae21107 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.tsfile.exception.NotImplementedException; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataInputStream; @@ -308,14 +309,16 @@ public List splitByPartition(IAnalysis analysis) { for (int i = 0; i < insertRowNodeList.size(); i++) { InsertRowNode insertRowNode = insertRowNodeList.get(i); // Data region for insert row node - // each row may belong to different database, pass null for auto-detection + // Each row may belong to different database. + final IDeviceID deviceID = insertRowNode.targetPath.getIDeviceIDAsFullDevice(); + final String databaseName = getDatabaseName(analysis, deviceID); TRegionReplicaSet dataRegionReplicaSet = analysis .getDataPartitionInfo() .getDataRegionReplicaSetForWriting( - insertRowNode.targetPath.getIDeviceIDAsFullDevice(), - TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime()), - null); + deviceID, + TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime(), databaseName), + databaseName); // Collect redirectInfo redirectInfo.add(dataRegionReplicaSet.getDataNodeLocations().get(0).getClientRpcEndPoint()); InsertRowsNode tmpNode = splitMap.get(dataRegionReplicaSet); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index cee1e325df228..0af35c0d72351 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -40,6 +40,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotImplementedException; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -188,18 +189,17 @@ public List splitByPartition(IAnalysis analysis) { Map>> splitMap = new HashMap<>(); Map>> splitMapForIndex = new HashMap<>(); + final IDeviceID deviceID = targetPath.getIDeviceIDAsFullDevice(); + final String databaseName = getDatabaseName(analysis, deviceID); for (int i = 0; i < insertRowNodeList.size(); i++) { InsertRowNode insertRowNode = insertRowNodeList.get(i); TTimePartitionSlot timePartitionSlot = - TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime()); + TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime(), databaseName); TRegionReplicaSet dataRegionReplicaSet = analysis .getDataPartitionInfo() - .getDataRegionReplicaSetForWriting( - targetPath.getIDeviceIDAsFullDevice(), - timePartitionSlot, - analysis.getDatabaseName()); + .getDataRegionReplicaSetForWriting(deviceID, timePartitionSlot, databaseName); Map> tmpMap = splitMap.computeIfAbsent(dataRegionReplicaSet, k -> new HashMap<>()); Map> tmpIndexMap = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 65e3207b7efe5..85ad303540230 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -236,15 +236,19 @@ public List splitByPartition(IAnalysis analysis) { return Collections.emptyList(); } - final Map deviceIDSplitInfoMap = collectSplitRanges(); + final String databaseName = getDatabaseName(analysis, getDeviceID(0)); + final Map deviceIDSplitInfoMap = + collectSplitRanges(databaseName); final Map> splitMap = - splitByReplicaSet(deviceIDSplitInfoMap, analysis); + splitByReplicaSet(deviceIDSplitInfoMap, analysis, databaseName); return doSplit(splitMap); } - private Map collectSplitRanges() { - long upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[0]); - TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[0]); + private Map collectSplitRanges(String database) { + long upperBoundOfTimePartition = + TimePartitionUtils.getTimePartitionUpperBound(times[0], database); + TTimePartitionSlot timePartitionSlot = + TimePartitionUtils.getTimePartitionSlot(times[0], database); int startLoc = 0; // included IDeviceID currDeviceId = getDeviceID(0); @@ -264,8 +268,9 @@ private Map collectSplitRanges() { splitInfo.timePartitionSlots.add(timePartitionSlot); // next init startLoc = i; - upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[i]); - timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i]); + upperBoundOfTimePartition = + TimePartitionUtils.getTimePartitionUpperBound(times[i], database); + timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i], database); currDeviceId = nextDeviceId; } } @@ -281,7 +286,9 @@ private Map collectSplitRanges() { } protected Map> splitByReplicaSet( - Map deviceIDSplitInfoMap, IAnalysis analysis) { + Map deviceIDSplitInfoMap, + IAnalysis analysis, + String databaseName) { Map> splitMap = new HashMap<>(); for (Entry entry : deviceIDSplitInfoMap.entrySet()) { @@ -291,7 +298,7 @@ protected Map> splitByReplicaSet( analysis .getDataPartitionInfo() .getDataRegionReplicaSetForWriting( - deviceID, splitInfo.timePartitionSlots, analysis.getDatabaseName()); + deviceID, splitInfo.timePartitionSlots, databaseName); splitInfo.replicaSets = replicaSets; // collect redirectInfo analysis.addEndPointToRedirectNodeList( @@ -387,17 +394,20 @@ protected WritePlanNode generateOneSplit(Map.Entry getTimePartitionSlots() { + public List getTimePartitionSlots(String database) { List result = new ArrayList<>(); - long upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[0]); - TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[0]); + long upperBoundOfTimePartition = + TimePartitionUtils.getTimePartitionUpperBound(times[0], database); + TTimePartitionSlot timePartitionSlot = + TimePartitionUtils.getTimePartitionSlot(times[0], database); for (int i = 1; i < times.length; i++) { // times are sorted in session API. if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( times[i], timePartitionSlot.getStartTime(), upperBoundOfTimePartition)) { result.add(timePartitionSlot); // next init - upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[i]); - timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i]); + upperBoundOfTimePartition = + TimePartitionUtils.getTimePartitionUpperBound(times[i], database); + timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i], database); } } result.add(timePartitionSlot); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java index 4a98819c6487f..0ef2d8850fd0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java @@ -206,7 +206,8 @@ public List splitByPartition(IAnalysis analysis) { .getDataPartitionInfo() .getDataRegionReplicaSetForWriting( insertRowNode.getDeviceID(), - TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime()), + TimePartitionUtils.getTimePartitionSlot( + insertRowNode.getTime(), analysis.getDatabaseName()), analysis.getDatabaseName()); // Collect redirectInfo diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index 029c55499bf9d..7abc37ffff03a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -264,7 +264,9 @@ protected InsertTabletNode getEmptySplit(int count) { } protected Map> splitByReplicaSet( - final Map deviceIDSplitInfoMap, final IAnalysis analysis) { + final Map deviceIDSplitInfoMap, + final IAnalysis analysis, + final String databaseName) { final Map> splitMap = new HashMap<>(); final Map endPointMap = new HashMap<>(); @@ -275,7 +277,7 @@ protected Map> splitByReplicaSet( analysis .getDataPartitionInfo() .getDataRegionReplicaSetForWriting( - deviceID, splitInfo.timePartitionSlots, analysis.getDatabaseName()); + deviceID, splitInfo.timePartitionSlots, databaseName); splitInfo.replicaSets = replicaSets; // collect redirectInfo endPointMap.put( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java index 0f0f92ef0fffe..fa4302e096eb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java @@ -367,6 +367,10 @@ public void updateLastCacheIfExists( final LastCacheUpdateSource updateSource, final boolean isAligned, final IMeasurementSchema[] measurementSchemas) { + if (!ClusterPartitionFetcher.getInstance().needLastCache(database)) { + return; + } + tableDeviceSchemaCache.updateLastCache( database, deviceID, measurements, updateSource, isAligned, measurementSchemas); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 66f6da892f3e8..6351984b7ddff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -3015,6 +3015,7 @@ private List getDeviceReplicaSets( IDeviceID deviceId, Filter timeFilter, Map> cachedSeriesSlotWithRegions) { + String database = dataPartition.getDatabaseNameByDevice(deviceId); // given seriesPartitionSlot has already been calculated final TSeriesPartitionSlot seriesPartitionSlot = dataPartition.calculateDeviceGroupId(deviceId); @@ -3034,7 +3035,8 @@ private List getDeviceReplicaSets( } if (timeSlotMap.size() == 1) { TTimePartitionSlot timePartitionSlot = timeSlotMap.keySet().iterator().next(); - if (TimePartitionUtils.satisfyPartitionStartTime(timeFilter, timePartitionSlot.startTime)) { + if (TimePartitionUtils.satisfyPartitionStartTime( + timeFilter, timePartitionSlot.startTime, database)) { cachedSeriesSlotWithRegions.put( seriesPartitionSlot.getSlotId(), timeSlotMap.values().iterator().next()); return timeSlotMap.values().iterator().next(); @@ -3047,7 +3049,8 @@ private List getDeviceReplicaSets( Set resultSet = new HashSet<>(); for (Map.Entry> entry : timeSlotMap.entrySet()) { TTimePartitionSlot timePartitionSlot = entry.getKey(); - if (TimePartitionUtils.satisfyPartitionStartTime(timeFilter, timePartitionSlot.startTime)) { + if (TimePartitionUtils.satisfyPartitionStartTime( + timeFilter, timePartitionSlot.startTime, database)) { resultSet.addAll(entry.getValue()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 8bd7f37423218..f14dcc4780529 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -1379,7 +1379,7 @@ private DataPartition fetchDataPartitionByDeviceDataSet( final DeviceEntryDataSet deviceEntryDataSet, final Filter globalTimeFilter) { final Pair, Pair> res = - getTimePartitionSlotList(globalTimeFilter, queryContext); + getTimePartitionSlotList(globalTimeFilter, queryContext, database); // there is no satisfied time range if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/DataPartitionBatchFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/DataPartitionBatchFetcher.java index d7e5c5f199337..a2dd13f6deb2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/DataPartitionBatchFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/DataPartitionBatchFetcher.java @@ -62,6 +62,12 @@ void setDatabase(final String database) { this.database = database; } + void ensureDatabaseTimePartitionConfig() { + if (database != null) { + fetcher.ensureDatabaseTimePartitionConfig(Collections.singleton(database)); + } + } + String getDatabase() { return database; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index ea4bf8679187f..820db71471b0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -360,6 +360,7 @@ public void start() { final String userName = queryContext.getSession().getUserName(); partitionFetcher.setDatabase(getPartitionQueryDatabase(node, isGeneratedByPipe)); + partitionFetcher.ensureDatabaseTimePartitionConfig(); if (!processSingleNode(node, i, totalFiles, userName)) { allFilesSucceeded = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java index aa0ec86cddd07..3efaca3722fa5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java @@ -184,7 +184,7 @@ private boolean firstPhase(LoadSingleTsFileNode node) { this::dispatchOnePieceNode); try { new TsFileSplitter(node.getTsFileResource().getTsFile(), pipeline) - .splitTsFileByDataPartition(); + .splitTsFileByDataPartition(node.getDatabase()); return pipeline.sendAllTsFileData(); } catch (IllegalStateException e) { LOGGER.warn( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index 0c33daa295f2d..4399978505403 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -166,8 +166,8 @@ public void fillValues(ByteBuffer buffer) throws QueryProcessException { } } - public TTimePartitionSlot getTimePartitionSlot() { - return TimePartitionUtils.getTimePartitionSlot(time); + public TTimePartitionSlot getTimePartitionSlot(String database) { + return TimePartitionUtils.getTimePartitionSlot(time, database); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java index 80dd52eca140e..f64603cde49c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java @@ -99,7 +99,8 @@ public List getTimePartitionSlots() { Set timePartitionSlotSet = new HashSet<>(); for (InsertRowStatement insertRowStatement : insertRowStatementList) { timePartitionSlotSet.add( - TimePartitionUtils.getTimePartitionSlot(insertRowStatement.getTime())); + TimePartitionUtils.getTimePartitionSlot( + insertRowStatement.getTime(), getDatabaseName(insertRowStatement))); } return new ArrayList<>(timePartitionSlotSet); } @@ -236,4 +237,11 @@ public Optional getDatabaseName() { protected void subRemoveAttributeColumns(List columnsToKeep) { insertRowStatementList.forEach(InsertRowStatement::removeAttributeColumns); } + + public static String getDatabaseName(final InsertBaseStatement statement) { + if (statement.getDatabaseName().isPresent()) { + return statement.getDatabaseName().get(); + } + return null; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 7f9d7ce5551a1..ea588f97b9594 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -235,25 +235,28 @@ public boolean isEmpty() { || columns.length == 0; } - public List getTimePartitionSlots() { + public List getTimePartitionSlots(String database) { List result = new ArrayList<>(); - long upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[0]); - TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[0]); + long upperBoundOfTimePartition = + TimePartitionUtils.getTimePartitionUpperBound(times[0], database); + TTimePartitionSlot timePartitionSlot = + TimePartitionUtils.getTimePartitionSlot(times[0], database); for (int i = 1; i < times.length; i++) { // times are sorted in session API. if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( times[i], timePartitionSlot.getStartTime(), upperBoundOfTimePartition)) { result.add(timePartitionSlot); // next init - upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[i]); - timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i]); + upperBoundOfTimePartition = + TimePartitionUtils.getTimePartitionUpperBound(times[i], database); + timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i], database); } } result.add(timePartitionSlot); return result; } - public TTimePartitionSlot getTimePartitionSlot(int i) { - return TimePartitionUtils.getTimePartitionSlot(times[i]); + public TTimePartitionSlot getTimePartitionSlot(int i, String database) { + return TimePartitionUtils.getTimePartitionSlot(times[i], database); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java index dc18d56055649..997a97092c1be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java @@ -36,6 +36,7 @@ public class DatabaseSchemaStatement extends Statement implements IConfigStateme private PartialPath databasePath; private Long ttl = null; private Long timePartitionInterval = null; + private Long timePartitionOrigin = null; private Integer maxSchemaRegionGroupNum = null; private Integer maxDataRegionGroupNum = null; private boolean enablePrintExceptionLog = true; @@ -96,6 +97,14 @@ public void setTimePartitionInterval(final Long timePartitionInterval) { this.timePartitionInterval = timePartitionInterval; } + public Long getTimePartitionOrigin() { + return timePartitionOrigin; + } + + public void setTimePartitionOrigin(final Long timePartitionOrigin) { + this.timePartitionOrigin = timePartitionOrigin; + } + public Integer getMaxSchemaRegionGroupNum() { return maxSchemaRegionGroupNum; } @@ -167,6 +176,8 @@ public String toString() { + dataReplicationFactor + ", timePartitionInterval=" + timePartitionInterval + + ", timePartitionOrigin=" + + timePartitionOrigin + ", maxSchemaRegionGroupNum=" + maxSchemaRegionGroupNum + ", maxDataRegionGroupNum=" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index f66a76c9f951b..9e1cbca6cef2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -322,7 +322,6 @@ public Map> getLocalDataRegionInfo() { @Override public void start() throws StartupException { recoverDataRegionNum = 0; - // build time Interval to divide time partition initTimePartition(); // create systemDir try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 3c94440301dda..72a575cd46c81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -841,11 +841,11 @@ private void recover() throws DataRegionException { dataRegionRecoveryContext.recoverPerformers) { recoverUnsealedTsFileCallBack(recoverPerformer); } - for (TsFileResource resource : tsFileManager.getTsFileList(true)) { + for (TsFileResource resource : tsFileManager.getTsFileList(true, databaseName)) { long partitionNum = resource.getTimePartition(); updatePartitionFileVersion(partitionNum, resource.getVersion()); } - for (TsFileResource resource : tsFileManager.getTsFileList(false)) { + for (TsFileResource resource : tsFileManager.getTsFileList(false, databaseName)) { long partitionNum = resource.getTimePartition(); updatePartitionFileVersion(partitionNum, resource.getVersion()); } @@ -1279,7 +1279,8 @@ public void insert(InsertRowNode insertRowNode) throws WriteProcessException { } // init map - long timePartitionId = TimePartitionUtils.getTimePartitionId(insertRowNode.getTime()); + long timePartitionId = + TimePartitionUtils.getTimePartitionId(insertRowNode.getTime(), databaseName); initFlushTimeMap(timePartitionId); insertRowNode.setLastFragment(true); @@ -1345,7 +1346,7 @@ private void split( int before = loc; long beforeTime = insertTabletNode.getTimes()[before]; // before time partition - long beforeTimePartition = TimePartitionUtils.getTimePartitionId(beforeTime); + long beforeTimePartition = TimePartitionUtils.getTimePartitionId(beforeTime, databaseName); // init flush time map initFlushTimeMap(beforeTimePartition); @@ -1353,7 +1354,7 @@ private void split( boolean isSequence = false; while (loc < endOffset) { long time = insertTabletNode.getTimes()[loc]; - final long timePartitionId = TimePartitionUtils.getTimePartitionId(time); + final long timePartitionId = TimePartitionUtils.getTimePartitionId(time, databaseName); long lastFlushTime; // judge if we should insert sequence @@ -1976,7 +1977,8 @@ private InsertRowsExecutionResult insertRowsWithTypeConsistencyCheck( return executionResult; } catch (DataTypeInconsistentException e) { InsertRowNode firstRow = subInsertRowsNode.getInsertRowNodeList().get(0); - long timePartitionId = TimePartitionUtils.getTimePartitionId(firstRow.getTime()); + long timePartitionId = + TimePartitionUtils.getTimePartitionId(firstRow.getTime(), databaseName); // flush both MemTables so that the new type can be inserted into a new MemTable flushWorkingProcessorsForTimePartition(timePartitionId); return retryInsertRowsAfterFlush(subInsertRowsNode, timePartitionId, infoForMetrics); @@ -2436,14 +2438,14 @@ public void deleteDALFolderAndClose() { /** close all tsfile resource */ public void closeAllResources() { - for (TsFileResource tsFileResource : tsFileManager.getTsFileList(false)) { + for (TsFileResource tsFileResource : tsFileManager.getTsFileList(false, databaseName)) { try { tsFileResource.close(); } catch (IOException e) { logger.error(StorageEngineMessages.CANNOT_CLOSE_TSFILE_RESOURCE, tsFileResource, e); } } - for (TsFileResource tsFileResource : tsFileManager.getTsFileList(true)) { + for (TsFileResource tsFileResource : tsFileManager.getTsFileList(true, databaseName)) { try { tsFileResource.close(); } catch (IOException e) { @@ -2465,8 +2467,8 @@ public void syncDeleteDataFiles() throws TsFileProcessorException { // normally, mergingModification is just need to be closed by after a merge task is finished. // we close it here just for IT test. closeAllResources(); - List tsFileResourceList = tsFileManager.getTsFileList(true); - tsFileResourceList.addAll(tsFileManager.getTsFileList(false)); + List tsFileResourceList = tsFileManager.getTsFileList(true, databaseName); + tsFileResourceList.addAll(tsFileManager.getTsFileList(false, databaseName)); tsFileResourceList.forEach( x -> { FileMetrics.getInstance().deleteTsFile(Collections.singletonList(x)); @@ -2964,7 +2966,7 @@ public QueryDataSource query( throws QueryProcessException { Pair, List> pair = - tsFileManager.getAllTsFileListForQuery(timePartitions, globalTimeFilter); + tsFileManager.getAllTsFileListForQuery(timePartitions, globalTimeFilter, databaseName); List seqTsFileResouceList = pair.left; List unSeqTsFileResouceList = pair.right; @@ -3136,7 +3138,7 @@ public IQueryDataSource queryForSeriesRegionScan( List timePartitions, long waitForLockTimeInMs) { Pair, List> pair = - tsFileManager.getAllTsFileListForQuery(timePartitions, globalTimeFilter); + tsFileManager.getAllTsFileListForQuery(timePartitions, globalTimeFilter, databaseName); List seqTsFileResouceList = pair.left; List unSeqTsFileResouceList = pair.right; @@ -3161,7 +3163,7 @@ public IQueryDataSource queryForSeriesRegionScan( List unSeqFileScanHandles = getFileHandleListForQuery( - tsFileManager.getTsFileList(false, timePartitions, globalTimeFilter), + tsFileManager.getTsFileList(false, timePartitions, globalTimeFilter, databaseName), pathList, queryContext, globalTimeFilter, @@ -3214,7 +3216,7 @@ public IQueryDataSource queryForDeviceRegionScan( long waitForLockTimeInMs) { Pair, List> pair = - tsFileManager.getAllTsFileListForQuery(timePartitions, globalTimeFilter); + tsFileManager.getAllTsFileListForQuery(timePartitions, globalTimeFilter, databaseName); List seqTsFileResouceList = pair.left; List unSeqTsFileResouceList = pair.right; @@ -3239,7 +3241,7 @@ public IQueryDataSource queryForDeviceRegionScan( List unSeqFileScanHandles = getFileHandleListForQuery( - tsFileManager.getTsFileList(false, timePartitions, globalTimeFilter), + tsFileManager.getTsFileList(false, timePartitions, globalTimeFilter, databaseName), devicePathToAligned, queryContext, globalTimeFilter, @@ -3385,8 +3387,9 @@ private void getTwoKindsOfTsFiles( List unsealedResource, long startTime, long endTime) { - List tsFileResources = tsFileManager.getTsFileList(true, startTime, endTime); - tsFileResources.addAll(tsFileManager.getTsFileList(false, startTime, endTime)); + List tsFileResources = + tsFileManager.getTsFileList(true, startTime, endTime, databaseName); + tsFileResources.addAll(tsFileManager.getTsFileList(false, startTime, endTime, databaseName)); tsFileResources.stream().filter(TsFileResource::isClosed).forEach(sealedResource::add); tsFileResources.stream() .filter(resource -> !resource.isClosed()) @@ -3676,12 +3679,14 @@ private List logDeletionInWAL(RelationalDeleteDataNode deleteD long startTime = modEntry.getStartTime(); long endTime = modEntry.getEndTime(); for (Map.Entry entry : workSequenceTsFileProcessors.entrySet()) { - if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { + if (TimePartitionUtils.satisfyPartitionId( + startTime, endTime, entry.getKey(), deleteDataNode.getDatabaseName())) { involvedProcessors.add(entry.getValue()); } } for (Map.Entry entry : workUnsequenceTsFileProcessors.entrySet()) { - if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { + if (TimePartitionUtils.satisfyPartitionId( + startTime, endTime, entry.getKey(), deleteDataNode.getDatabaseName())) { involvedProcessors.add(entry.getValue()); } } @@ -3728,7 +3733,8 @@ private List logDeletionInWAL( if (TimePartitionUtils.satisfyPartitionId( templateDeleteDataNode.getDeleteStartTime(), templateDeleteDataNode.getDeleteEndTime(), - entry.getKey())) { + entry.getKey(), + databaseName)) { WALFlushListener walFlushListener = entry.getValue().logDeleteDataNodeInWAL(deleteDataNode); walFlushListeners.add(walFlushListener); } @@ -3737,7 +3743,8 @@ private List logDeletionInWAL( if (TimePartitionUtils.satisfyPartitionId( templateDeleteDataNode.getDeleteStartTime(), templateDeleteDataNode.getDeleteEndTime(), - entry.getKey())) { + entry.getKey(), + databaseName)) { WALFlushListener walFlushListener = entry.getValue().logDeleteDataNodeInWAL(deleteDataNode); walFlushListeners.add(walFlushListener); } @@ -5041,7 +5048,7 @@ private TsFileResource unloadTsFileInside(File fileToBeUnloaded) { writeLock("unloadTsFileInside"); TsFileResource unloadedTsFileResource = null; try { - Iterator sequenceIterator = tsFileManager.getIterator(true); + Iterator sequenceIterator = tsFileManager.getIterator(true, databaseName); while (sequenceIterator.hasNext()) { TsFileResource sequenceResource = sequenceIterator.next(); if (sequenceResource.getTsFile().getName().equals(fileToBeUnloaded.getName())) { @@ -5052,7 +5059,8 @@ private TsFileResource unloadTsFileInside(File fileToBeUnloaded) { } } if (unloadedTsFileResource == null) { - Iterator unsequenceIterator = tsFileManager.getIterator(false); + Iterator unsequenceIterator = + tsFileManager.getIterator(false, databaseName); while (unsequenceIterator.hasNext()) { TsFileResource unsequenceResource = unsequenceIterator.next(); if (unsequenceResource.getTsFile().getName().equals(fileToBeUnloaded.getName())) { @@ -5080,11 +5088,11 @@ public Collection getWorkUnsequenceTsFileProcessors() { } public List getSequenceFileList() { - return tsFileManager.getTsFileList(true); + return tsFileManager.getTsFileList(true, databaseName); } public List getUnSequenceFileList() { - return tsFileManager.getTsFileList(false); + return tsFileManager.getTsFileList(false, databaseName); } @Override @@ -5165,7 +5173,8 @@ public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode) continue; } // init map - long timePartitionId = TimePartitionUtils.getTimePartitionId(insertRowNode.getTime()); + long timePartitionId = + TimePartitionUtils.getTimePartitionId(insertRowNode.getTime(), databaseName); if (config.isEnableSeparateData() && !lastFlushTimeMap.checkAndCreateFlushedTimePartition(timePartitionId, true)) { @@ -5286,7 +5295,8 @@ public void insert(InsertRowsNode insertRowsNode) continue; } // init map - timePartitionIds[i] = TimePartitionUtils.getTimePartitionId(insertRowNode.getTime()); + timePartitionIds[i] = + TimePartitionUtils.getTimePartitionId(insertRowNode.getTime(), databaseName); if (config.isEnableSeparateData() && !lastFlushTimeMap.checkAndCreateFlushedTimePartition(timePartitionIds[i], true)) { @@ -5484,14 +5494,14 @@ public void addSettleFilesToList( List unseqResourcesToBeSettled, List tsFilePaths) { if (tsFilePaths.isEmpty()) { - for (TsFileResource resource : tsFileManager.getTsFileList(true)) { + for (TsFileResource resource : tsFileManager.getTsFileList(true, databaseName)) { if (!resource.isClosed()) { continue; } resource.setSettleTsFileCallBack(this::settleTsFileCallBack); seqResourcesToBeSettled.add(resource); } - for (TsFileResource resource : tsFileManager.getTsFileList(false)) { + for (TsFileResource resource : tsFileManager.getTsFileList(false, databaseName)) { if (!resource.isClosed()) { continue; } @@ -5509,7 +5519,7 @@ public void addSettleFilesToList( .getParentFile() .getParentFile() .getName())) { - for (TsFileResource resource : tsFileManager.getTsFileList(true)) { + for (TsFileResource resource : tsFileManager.getTsFileList(true, databaseName)) { if (resource.getTsFile().getAbsolutePath().equals(tsFilePath)) { resource.setSettleTsFileCallBack(this::settleTsFileCallBack); seqResourcesToBeSettled.add(resource); @@ -5517,7 +5527,7 @@ public void addSettleFilesToList( } } } else { - for (TsFileResource resource : tsFileManager.getTsFileList(false)) { + for (TsFileResource resource : tsFileManager.getTsFileList(false, databaseName)) { if (resource.getTsFile().getAbsolutePath().equals(tsFilePath)) { unseqResourcesToBeSettled.add(resource); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java index 5b8f5491a39ed..b85d3b9a10112 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java @@ -199,11 +199,12 @@ public static boolean clearSnapshotOfDataRegion(DataRegion dataRegion) { } private void readLockTheFile() { + String database = dataRegion.getDatabaseName(); TsFileManager manager = dataRegion.getTsFileManager(); manager.readLock(); try { - seqFiles = manager.getTsFileList(true); - unseqFiles = manager.getTsFileList(false); + seqFiles = manager.getTsFileList(true, database); + unseqFiles = manager.getTsFileList(false, database); for (TsFileResource resource : seqFiles) { resource.readLock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java index b2cb788601091..86dcec5a5b6e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java @@ -66,7 +66,11 @@ public TsFileManager(String storageGroupName, String dataRegionId, String dataRe } public List getTsFileList(boolean sequence) { - return getTsFileList(sequence, null, null); + return getTsFileList(sequence, storageGroupName); + } + + public List getTsFileList(boolean sequence, String database) { + return getTsFileList(sequence, null, null, database); } /** @@ -75,6 +79,15 @@ public List getTsFileList(boolean sequence) { */ public List getTsFileList( boolean sequence, List timePartitions, Filter timeFilter) { + return getTsFileList(sequence, timePartitions, timeFilter, storageGroupName); + } + + /** + * @param sequence {@code true} for sequence, {@code false} for unsequence + * @param timePartitions {@code null} for all time partitions, empty for zero time partitions + */ + public List getTsFileList( + boolean sequence, List timePartitions, Filter timeFilter, String database) { // the iteration of ConcurrentSkipListMap is not concurrent secure // so we must add read lock here readLock(); @@ -83,7 +96,7 @@ public List getTsFileList( Map chosenMap = sequence ? sequenceFiles : unsequenceFiles; if (timePartitions == null) { for (Map.Entry entry : chosenMap.entrySet()) { - if (TimePartitionUtils.satisfyTimePartition(timeFilter, entry.getKey())) { + if (TimePartitionUtils.satisfyTimePartition(timeFilter, entry.getKey(), database)) { allResources.addAll(entry.getValue().getArrayList()); } } @@ -108,16 +121,26 @@ public List getTsFileList( */ public Pair, List> getAllTsFileListForQuery( List timePartitions, Filter timeFilter) { + return getAllTsFileListForQuery(timePartitions, timeFilter, storageGroupName); + } + + /** + * don't need to acquire lock again, caller should guarantee the lock has been acquired + * + * @return left is seq resource list, right is unSeq resource list + */ + public Pair, List> getAllTsFileListForQuery( + List timePartitions, Filter timeFilter, String database) { List seq = new ArrayList<>(); List unSeq = new ArrayList<>(); if (timePartitions == null) { for (Map.Entry entry : sequenceFiles.entrySet()) { - if (TimePartitionUtils.satisfyTimePartition(timeFilter, entry.getKey())) { + if (TimePartitionUtils.satisfyTimePartition(timeFilter, entry.getKey(), database)) { seq.addAll(entry.getValue().getArrayList()); } } for (Map.Entry entry : unsequenceFiles.entrySet()) { - if (TimePartitionUtils.satisfyTimePartition(timeFilter, entry.getKey())) { + if (TimePartitionUtils.satisfyTimePartition(timeFilter, entry.getKey(), database)) { unSeq.addAll(entry.getValue().getArrayList()); } } @@ -159,6 +182,11 @@ public Pair, List> getTsFileListSnapshot( } public List getTsFileList(boolean sequence, long startTime, long endTime) { + return getTsFileList(sequence, startTime, endTime, storageGroupName); + } + + public List getTsFileList( + boolean sequence, long startTime, long endTime, String database) { // the iteration of ConcurrentSkipListMap is not concurrent secure // so we must add read lock here readLock(); @@ -166,7 +194,7 @@ public List getTsFileList(boolean sequence, long startTime, long List allResources = new ArrayList<>(); Map chosenMap = sequence ? sequenceFiles : unsequenceFiles; for (Map.Entry entry : chosenMap.entrySet()) { - if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { + if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey(), database)) { allResources.addAll(entry.getValue().getArrayList()); } } @@ -195,9 +223,13 @@ public TsFileResourceList getOrCreateUnsequenceListByTimePartition(long timePart } public Iterator getIterator(boolean sequence) { + return getIterator(sequence, storageGroupName); + } + + public Iterator getIterator(boolean sequence, String database) { readLock(); try { - return getTsFileList(sequence).iterator(); + return getTsFileList(sequence, database).iterator(); } finally { readUnlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 85778719a354b..fc2e56beced7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -1152,12 +1152,7 @@ public long getTimePartition() { * @throws PartitionViolationException if the data of the file spans partitions or it is empty */ public long getTimePartitionWithCheck() throws PartitionViolationException { - return timeIndex.getTimePartitionWithCheck(file.toString()); - } - - /** Check whether the tsFile spans multiple time partitions. */ - public boolean isSpanMultiTimePartitions() { - return timeIndex.isSpanMultiTimePartitions(); + return timeIndex.getTimePartitionWithCheck(file.toString(), getDatabaseName()); } public void setExclusiveModFile(ModificationFile exclusiveModFile) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java index 8af037a5924ef..7305cad57db70 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java @@ -269,11 +269,11 @@ private long[] enLargeArray(long[] array, long defaultValue) { } @Override - public long getTimePartition(String tsFilePath) { + public long getTimePartition(String tsFilePath, String database) { try { if (deviceToIndex != null && !deviceToIndex.isEmpty()) { return TimePartitionUtils.getTimePartitionId( - startTimes[deviceToIndex.values().iterator().next()]); + startTimes[deviceToIndex.values().iterator().next()], database); } String[] filePathSplits = FilePathUtils.splitTsFilePath(tsFilePath); return Long.parseLong(filePathSplits[filePathSplits.length - 2]); @@ -283,30 +283,33 @@ public long getTimePartition(String tsFilePath) { } @Override - public long getTimePartitionWithCheck(String tsFilePath) throws PartitionViolationException { + public long getTimePartitionWithCheck(String tsFilePath, String database) + throws PartitionViolationException { try { - return getTimePartitionWithCheck(); + return getTimePartitionWithCheck(database); } catch (PartitionViolationException e) { throw new PartitionViolationException(tsFilePath); } } @Override - public boolean isSpanMultiTimePartitions() { + public boolean isSpanMultiTimePartitions(String database) { try { - getTimePartitionWithCheck(); + getTimePartitionWithCheck(database); return false; } catch (PartitionViolationException e) { return true; } } - private long getTimePartitionWithCheck() throws PartitionViolationException { + private long getTimePartitionWithCheck(String database) throws PartitionViolationException { Long partitionId = null; for (final int index : deviceToIndex.values()) { - final long startTimePartitionId = TimePartitionUtils.getTimePartitionId(startTimes[index]); - final long endTimePartitionId = TimePartitionUtils.getTimePartitionId(endTimes[index]); + final long startTimePartitionId = + TimePartitionUtils.getTimePartitionId(startTimes[index], database); + final long endTimePartitionId = + TimePartitionUtils.getTimePartitionId(endTimes[index], database); if (startTimePartitionId != endTimePartitionId) { throw new PartitionViolationException(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index f6f598ef9a443..68586247ea721 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -204,7 +204,7 @@ public long calculateRamSize() { } @Override - public long getTimePartition(String tsFilePath) { + public long getTimePartition(String tsFilePath, String database) { try { String[] filePathSplits = FilePathUtils.splitTsFilePath(tsFilePath); return Long.parseLong(filePathSplits[filePathSplits.length - 2]); @@ -214,9 +214,10 @@ public long getTimePartition(String tsFilePath) { } @Override - public long getTimePartitionWithCheck(String tsFilePath) throws PartitionViolationException { - final long startPartitionId = TimePartitionUtils.getTimePartitionId(startTime); - final long endPartitionId = TimePartitionUtils.getTimePartitionId(endTime); + public long getTimePartitionWithCheck(String tsFilePath, String database) + throws PartitionViolationException { + final long startPartitionId = TimePartitionUtils.getTimePartitionId(startTime, database); + final long endPartitionId = TimePartitionUtils.getTimePartitionId(endTime, database); if (startPartitionId == endPartitionId) { return startPartitionId; @@ -226,9 +227,9 @@ public long getTimePartitionWithCheck(String tsFilePath) throws PartitionViolati } @Override - public boolean isSpanMultiTimePartitions() { - return TimePartitionUtils.getTimePartitionId(startTime) - != TimePartitionUtils.getTimePartitionId(endTime); + public boolean isSpanMultiTimePartitions(String database) { + return TimePartitionUtils.getTimePartitionId(startTime, database) + != TimePartitionUtils.getTimePartitionId(endTime, database); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java index b5110786943c6..43d1dce1200be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java @@ -103,25 +103,29 @@ ITimeIndex deserialize(InputStream inputStream, IDeviceID.Deserializer deseriali * get time partition * * @param tsFilePath tsFile absolute path + * @param database database name * @return partition */ - long getTimePartition(String tsFilePath); + long getTimePartition(String tsFilePath, String database); /** * get time partition with check. If data of tsFile spans partitions, an exception will be thrown * * @param tsFilePath tsFile path + * @param database database name * @return partition * @throws PartitionViolationException data of tsFile spans partitions */ - long getTimePartitionWithCheck(String tsFilePath) throws PartitionViolationException; + long getTimePartitionWithCheck(String tsFilePath, String database) + throws PartitionViolationException; /** * Check whether the tsFile spans multiple time partitions. * + * @param database database name * @return true if the tsFile spans multiple time partitions, otherwise false. */ - boolean isSpanMultiTimePartitions(); + boolean isSpanMultiTimePartitions(String database); /** * update start time diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java index 370e6854c5f4d..65d465547666b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java @@ -66,6 +66,7 @@ public class AlignedChunkData implements ChunkData { // ---------------- Permanent state: describes this ChunkData and its final result // ---------------- protected final TTimePartitionSlot timePartitionSlot; + protected long timePartitionEndTime; protected final IDeviceID device; protected final EncryptParameter encryptParameter; protected List chunkHeaderList; @@ -159,6 +160,7 @@ protected AlignedChunkData(final AlignedChunkData alignedChunkData) { alignedChunkData.device, alignedChunkData.timePartitionSlot, alignedChunkData.encryptParameter); + this.timePartitionEndTime = alignedChunkData.timePartitionEndTime; this.satisfiedLengthQueue = new LinkedList<>(alignedChunkData.satisfiedLengthQueue); this.needDecodeChunk = alignedChunkData.needDecodeChunk; addAttrDataSize(); @@ -178,6 +180,8 @@ protected AlignedChunkData( this.encryptParameter = encryptParameter; this.chunkHeaderList = new ArrayList<>(); this.timePartitionSlot = timePartitionSlot; + this.timePartitionEndTime = + TimePartitionUtils.getTimePartitionEndTime(timePartitionSlot.getStartTime()); this.needDecodeChunk = true; this.pageNumbers = new ArrayList<>(); this.satisfiedLengthQueue = new LinkedList<>(); @@ -198,6 +202,11 @@ public IDeviceID getDevice() { return device; } + @Override + public void setTimePartitionEndTime(long endTime) { + this.timePartitionEndTime = endTime; + } + @Override public TTimePartitionSlot getTimePartitionSlot() { return timePartitionSlot; @@ -448,7 +457,7 @@ protected boolean isInCurrentTimePartition(final long time) { // Use the overflow-safe end time. A naive "start + interval - 1" wraps around for the last // representable partition (and equals the start when interval == 1), which would either drop // the upper bound or exclude points that belong to the current partition. - final long partitionEnd = TimePartitionUtils.getTimePartitionEndTime(partitionStart); + final long partitionEnd = timePartitionEndTime; return time >= partitionStart && time <= partitionEnd; } @@ -566,7 +575,7 @@ private void writeChunkLayout(final DataOutputStream stream) throws IOException public static AlignedChunkData deserialize(final InputStream stream) throws IOException, PageException { final TTimePartitionSlot timePartitionSlot = - TimePartitionUtils.getTimePartitionSlot(ReadWriteIOUtils.readLong(stream)); + new TTimePartitionSlot(ReadWriteIOUtils.readLong(stream)); final boolean isStringArrayDeviceID = ReadWriteIOUtils.readBool(stream); final IDeviceID device = isStringArrayDeviceID diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java index 9db2c9590b2b6..c42c9852ff450 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.storageengine.load.splitter; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.exception.write.PageException; @@ -67,7 +66,7 @@ protected void encodeAndBuildChunks() throws IOException, PageException { final long startTime = timePartitionSlot.getStartTime(); // Overflow-safe partition end: "start + interval - 1" wraps around for the last representable // partition and equals the start when interval == 1. - final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); + final long endTime = timePartitionEndTime; // Process each standalone value chunk for (int i = 0; i < chunkHeaderList.size(); i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkData.java index c35e418971463..fdaaf6496026d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkData.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.load.splitter; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; @@ -56,6 +57,8 @@ public interface ChunkData extends TsFileData { boolean isAligned(); + void setTimePartitionEndTime(long endTime); + boolean isEntireChunk(); void setNotDecode(); @@ -128,6 +131,18 @@ static ChunkData createChunkData( : new NonAlignedChunkData(device, chunkHeader, timePartitionSlot); } + static ChunkData createChunkData( + boolean isAligned, + IDeviceID device, + ChunkHeader chunkHeader, + TTimePartitionSlot timePartitionSlot, + String database) { + ChunkData chunkData = createChunkData(isAligned, device, chunkHeader, timePartitionSlot); + chunkData.setTimePartitionEndTime( + TimePartitionUtils.getTimePartitionEndTime(timePartitionSlot.getStartTime(), database)); + return chunkData; + } + static ChunkData deserialize(InputStream stream) throws PageException, IOException { return ReadWriteIOUtils.readBool(stream) ? AlignedChunkData.deserialize(stream) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java index d5342c2918c87..00bd7c347cac0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java @@ -61,6 +61,7 @@ public class NonAlignedChunkData implements ChunkData { private List chunkPayloadRefs = new ArrayList<>(); private final TTimePartitionSlot timePartitionSlot; + protected long timePartitionEndTime; private final IDeviceID device; private final ChunkHeader chunkHeader; private final EncryptParameter encryptParameter; @@ -117,6 +118,8 @@ public NonAlignedChunkData( this.device = device; this.chunkHeader = chunkHeader; this.timePartitionSlot = timePartitionSlot; + this.timePartitionEndTime = + TimePartitionUtils.getTimePartitionEndTime(timePartitionSlot.getStartTime()); this.encryptParameter = encryptParameter; this.needDecodeChunk = true; addAttrDataSize(); @@ -134,6 +137,11 @@ public IDeviceID getDevice() { return device; } + @Override + public void setTimePartitionEndTime(long endTime) { + this.timePartitionEndTime = endTime; + } + @Override public TTimePartitionSlot getTimePartitionSlot() { return timePartitionSlot; @@ -227,7 +235,7 @@ private void encodeAndBuildChunk() throws IOException, PageException { for (PageBuffer page : pageBuffers) { if (page.needDecode) { final long partitionStart = timePartitionSlot.getStartTime(); - final long partitionEnd = getTimePartitionEnd(partitionStart); + final long partitionEnd = timePartitionEndTime; // Resolved once per page; the service has no bulk entry point, so the loop stays here. final TypeServices.ObjectValueChunkWriter valueChunkWriter = TypeServices.StorageEngine.OBJECT_VALUE_CHUNK_WRITER_SERVICE.call( @@ -272,12 +280,6 @@ private void encodeAndBuildChunk() throws IOException, PageException { this.dataSize += chunkData.remaining() + statistics.getSerializedSize(); } - private static long getTimePartitionEnd(final long partitionStart) { - // Delegate to the overflow-safe helper: a naive "start + interval - 1" wraps around for the - // last representable partition and collapses to the start itself when interval == 1. - return TimePartitionUtils.getTimePartitionEndTime(partitionStart); - } - // ----------------------- Serialization / Deserialization ----------------------- @Override @@ -361,7 +363,7 @@ private void writeChunkLayout(final DataOutputStream stream) throws IOException public static NonAlignedChunkData deserialize(final InputStream stream) throws IOException, PageException { final TTimePartitionSlot timePartitionSlot = - TimePartitionUtils.getTimePartitionSlot(ReadWriteIOUtils.readLong(stream)); + new TTimePartitionSlot(ReadWriteIOUtils.readLong(stream)); final boolean isStringArrayDeviceID = ReadWriteIOUtils.readBool(stream); final IDeviceID device = isStringArrayDeviceID diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java index 9b17746f942a7..75fcf452b6ffa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java @@ -71,6 +71,7 @@ public class TsFileSplitter { private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private final File tsFile; + private String database; private final TsFileDataConsumer consumer; private Map offset2ChunkMetadata = new HashMap<>(); private List deletions = new ArrayList<>(); @@ -101,12 +102,18 @@ private ChunkData createChunkData( final IDeviceID device, final ChunkHeader header, final TTimePartitionSlot timePartitionSlot) { - return ChunkData.createChunkData(aligned, device, header, timePartitionSlot); + return ChunkData.createChunkData(aligned, device, header, timePartitionSlot, database); } - @SuppressWarnings({"squid:S3776", "squid:S6541"}) public void splitTsFileByDataPartition() throws IOException, LoadFileException, IllegalStateException { + splitTsFileByDataPartition(null); + } + + @SuppressWarnings({"squid:S3776", "squid:S6541"}) + public void splitTsFileByDataPartition(String database) + throws IOException, LoadFileException, IllegalStateException { + this.database = database; try (TsFileSequenceReader reader = new TsFileSequenceReader(new BufferedTsFileInput(tsFile.toPath()), true, false, null)) { getAllModification(deletions); @@ -199,7 +206,7 @@ private void processTimeChunkOrNonAlignedChunk(TsFileSequenceReader reader, byte return; } TTimePartitionSlot timePartitionSlot = - TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getStartTime()); + TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getStartTime(), database); ChunkData chunkData = createChunkData(isAligned, curDevice, header, timePartitionSlot); if (!needDecodeChunk(chunkMetadata)) { @@ -251,7 +258,7 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( ? chunkMetadata.getStartTime() : pageHeader.getStartTime(); TTimePartitionSlot pageTimePartitionSlot = - TimePartitionUtils.getTimePartitionSlot(startTime); + TimePartitionUtils.getTimePartitionSlot(startTime, database); if (!timePartitionSlot.equals(pageTimePartitionSlot)) { if (!isAligned) { chunkData.endChunk(); @@ -284,10 +291,11 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( // partition, whose upper bound is Long.MAX_VALUE itself, so the helper is used: it keeps // the point that equals Long.MAX_VALUE inside its own partition. long endTime = - TimePartitionUtils.getTimePartitionUpperBound(timePartitionSlot.getStartTime()); + TimePartitionUtils.getTimePartitionUpperBound( + timePartitionSlot.getStartTime(), database); for (int i = 0; i < times.length; i++) { if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( - times[i], timePartitionSlot.getStartTime(), endTime)) { + times[i], timePartitionSlot.getStartTime(), endTime, database)) { chunkData.writeDecodePage(times, values, satisfiedLength); if (isAligned) { chunkData.endChunk(); @@ -299,10 +307,11 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( consumeChunkData(measurementId, chunkOffset, chunkData); } - timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i]); + timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i], database); satisfiedLength = 0; endTime = - TimePartitionUtils.getTimePartitionUpperBound(timePartitionSlot.getStartTime()); + TimePartitionUtils.getTimePartitionUpperBound( + timePartitionSlot.getStartTime(), database); chunkData = createChunkData(isAligned, curDevice, header, timePartitionSlot); } satisfiedLength += 1; @@ -531,17 +540,17 @@ private void consumeChunkData(String measurement, long offset, ChunkData chunkDa } private boolean needDecodeChunk(IChunkMetadata chunkMetadata) { - return !TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getStartTime()) - .equals(TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getEndTime())); + return !TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getStartTime(), database) + .equals(TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getEndTime(), database)); } private boolean needDecodePage(PageHeader pageHeader, IChunkMetadata chunkMetadata) { if (pageHeader.getStatistics() == null) { - return !TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getStartTime()) - .equals(TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getEndTime())); + return !TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getStartTime(), database) + .equals(TimePartitionUtils.getTimePartitionSlot(chunkMetadata.getEndTime(), database)); } - return !TimePartitionUtils.getTimePartitionSlot(pageHeader.getStartTime()) - .equals(TimePartitionUtils.getTimePartitionSlot(pageHeader.getEndTime())); + return !TimePartitionUtils.getTimePartitionSlot(pageHeader.getStartTime(), database) + .equals(TimePartitionUtils.getTimePartitionSlot(pageHeader.getEndTime(), database)); } private Pair decodePage( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java index 9dae1181295c8..886138b52375f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java @@ -119,7 +119,7 @@ public static void rewriteTsFile( throws IOException, WriteProcessException, IllegalPathException { try (TsFileSplitByPartitionTool rewriteTool = new TsFileSplitByPartitionTool(resourceToBeRewritten)) { - rewriteTool.parseAndRewriteFile(rewrittenResources); + rewriteTool.parseAndRewriteFile(rewrittenResources, resourceToBeRewritten.getDatabaseName()); } } @@ -134,7 +134,7 @@ public void close() throws IOException { * @throws IOException WriteProcessException */ @SuppressWarnings({"squid:S3776", "deprecation"}) // Suppress high Cognitive Complexity warning - public void parseAndRewriteFile(List rewrittenResources) + public void parseAndRewriteFile(List rewrittenResources, String database) throws IOException, WriteProcessException, IllegalPathException { // check if the TsFile has correct header if (!fileCheck()) { @@ -182,7 +182,8 @@ public void parseAndRewriteFile(List rewrittenResources) // a new Page PageHeader pageHeader = reader.readPageHeader(dataType, header.getChunkType() == MetaMarker.CHUNK_HEADER); - boolean needToDecode = checkIfNeedToDecode(measurementSchema, deviceId, pageHeader); + boolean needToDecode = + checkIfNeedToDecode(measurementSchema, deviceId, pageHeader, database); needToDecodeInfo.add(needToDecode); ByteBuffer pageData = !needToDecode @@ -199,7 +200,8 @@ public void parseAndRewriteFile(List rewrittenResources) pageHeadersInChunk, dataInChunk, needToDecodeInfo, - chunkHeaderOffset); + chunkHeaderOffset, + database); firstChunkInChunkGroup = false; break; case MetaMarker.OPERATION_INDEX_RANGE: @@ -250,7 +252,7 @@ public void parseAndRewriteFile(List rewrittenResources) * false. */ protected boolean checkIfNeedToDecode( - MeasurementSchema schema, IDeviceID deviceId, PageHeader pageHeader) { + MeasurementSchema schema, IDeviceID deviceId, PageHeader pageHeader, String database) { if (pageHeader.getStatistics() == null) { return true; } @@ -269,8 +271,8 @@ protected boolean checkIfNeedToDecode( } } } - return TimePartitionUtils.getTimePartitionId(pageHeader.getStartTime()) - != TimePartitionUtils.getTimePartitionId(pageHeader.getEndTime()); + return TimePartitionUtils.getTimePartitionId(pageHeader.getStartTime(), database) + != TimePartitionUtils.getTimePartitionId(pageHeader.getEndTime(), database); } /** @@ -285,17 +287,27 @@ protected void reWriteChunk( List pageHeadersInChunk, List pageDataInChunk, List needToDecodeInfoInChunk, - long chunkHeaderOffset) + long chunkHeaderOffset, + String database) throws IOException, PageException, IllegalPathException { valueDecoder = Decoder.getDecoderByType(schema.getEncodingType(), schema.getType()); Map partitionChunkWriterMap = new HashMap<>(); for (int i = 0; i < pageDataInChunk.size(); i++) { if (Boolean.TRUE.equals(needToDecodeInfoInChunk.get(i))) { decodeAndWritePage( - deviceId, schema, pageDataInChunk.get(i), partitionChunkWriterMap, chunkHeaderOffset); + deviceId, + schema, + pageDataInChunk.get(i), + partitionChunkWriterMap, + chunkHeaderOffset, + database); } else { writePage( - schema, pageHeadersInChunk.get(i), pageDataInChunk.get(i), partitionChunkWriterMap); + schema, + pageHeadersInChunk.get(i), + pageDataInChunk.get(i), + partitionChunkWriterMap, + database); } } for (Entry entry : partitionChunkWriterMap.entrySet()) { @@ -354,9 +366,10 @@ protected void writePage( MeasurementSchema schema, PageHeader pageHeader, ByteBuffer pageData, - Map partitionChunkWriterMap) + Map partitionChunkWriterMap, + String database) throws PageException { - long partitionId = TimePartitionUtils.getTimePartitionId(pageHeader.getStartTime()); + long partitionId = TimePartitionUtils.getTimePartitionId(pageHeader.getStartTime(), database); getOrDefaultTsFileIOWriter(oldTsFile, partitionId); ChunkWriterImpl chunkWriter = partitionChunkWriterMap.computeIfAbsent(partitionId, v -> new ChunkWriterImpl(schema)); @@ -368,7 +381,8 @@ protected void decodeAndWritePage( MeasurementSchema schema, ByteBuffer pageData, Map partitionChunkWriterMap, - long chunkHeaderOffset) + long chunkHeaderOffset, + String database) throws IOException, IllegalPathException { valueDecoder.reset(); PageReader pageReader = @@ -377,7 +391,7 @@ protected void decodeAndWritePage( List deleteIntervalList = getOldSortedDeleteIntervals(deviceId, schema); pageReader.setDeleteIntervalList(deleteIntervalList); BatchData batchData = pageReader.getAllSatisfiedPageData(); - rewritePageIntoFiles(batchData, schema, partitionChunkWriterMap); + rewritePageIntoFiles(batchData, schema, partitionChunkWriterMap, database); } private List getOldSortedDeleteIntervals( @@ -402,13 +416,14 @@ private List getOldSortedDeleteIntervals( protected void rewritePageIntoFiles( BatchData batchData, MeasurementSchema schema, - Map partitionChunkWriterMap) { + Map partitionChunkWriterMap, + String database) { final Type type = Type.fromTsDataType(schema.getType()); long previousPartitionId = 0; ChunkWriterImpl chunkWriter = null; while (batchData.hasCurrent()) { long time = batchData.currentTime(); - long partitionId = TimePartitionUtils.getTimePartitionId(time); + long partitionId = TimePartitionUtils.getTimePartitionId(time, database); if (chunkWriter == null || partitionId != previousPartitionId) { previousPartitionId = partitionId; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/settle/TsFileAndModSettleTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/settle/TsFileAndModSettleTool.java index 9f4727b0d9145..c9fe373b04b5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/settle/TsFileAndModSettleTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/settle/TsFileAndModSettleTool.java @@ -222,7 +222,8 @@ public void settleOneTsFileAndMod( } try (TsFileSplitByPartitionTool tsFileRewriteTool = new TsFileSplitByPartitionTool(resourceToBeSettled)) { - tsFileRewriteTool.parseAndRewriteFile(settledResources); + tsFileRewriteTool.parseAndRewriteFile( + settledResources, resourceToBeSettled.getDatabaseName()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java index d935821cdba4e..1425fa187d957 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java @@ -21,6 +21,8 @@ import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.pipe.event.ProgressReportEvent; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEventFactory; @@ -34,6 +36,29 @@ public class PipeRealtimeDataRegionSourceTest { private static final String TEST_REFERENCE_HOLDER = PipeRealtimeDataRegionSourceTest.class.getName(); + @Test + public void timePartitionBoundsRespectDatabaseOrigin() { + final String database = "pipe_origin.db"; + final TDatabaseSchema schema = new TDatabaseSchema(); + schema.setName(database); + schema.setTimePartitionOrigin(1050L); + schema.setTimePartitionInterval(100L); + TimePartitionUtils.updateDatabaseTimePartitionConfig(database, schema); + + try { + Assert.assertEquals( + 1L, PipeRealtimeDataRegionSource.getStartTimePartitionIdLowerBound(1150L, database)); + Assert.assertEquals( + 1L, PipeRealtimeDataRegionSource.getEndTimePartitionIdUpperBound(1150L, database)); + Assert.assertEquals( + 2L, PipeRealtimeDataRegionSource.getStartTimePartitionIdLowerBound(1151L, database)); + Assert.assertEquals( + 0L, PipeRealtimeDataRegionSource.getEndTimePartitionIdUpperBound(1151L, database)); + } finally { + TimePartitionUtils.removeDatabaseTimePartitionConfig(database); + } + } + @Test public void progressReportEventReleasesDroppedHeartbeatEvent() throws Exception { try (final ProgressReportTestSource source = new ProgressReportTestSource()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakePartitionFetcherImpl.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakePartitionFetcherImpl.java index 85c2673cc91b9..b227db9e804fc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakePartitionFetcherImpl.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakePartitionFetcherImpl.java @@ -32,6 +32,7 @@ import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; @@ -43,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class FakePartitionFetcherImpl implements IPartitionFetcher { @@ -303,8 +305,17 @@ public DataPartition getOrCreateDataPartition( partitionExecutor.getSeriesPartitionSlot(dataPartitionQueryParam.getDeviceID()); Map> timePartitionSlotListMap = sgPartitionMap.computeIfAbsent(seriesPartitionSlot, k -> new HashMap<>()); - for (TTimePartitionSlot timePartitionSlot : - dataPartitionQueryParam.getTimePartitionSlotList()) { + List timePartitionSlots = + dataPartitionQueryParam.hasTimeList() + ? dataPartitionQueryParam.getTimeList().stream() + .map( + time -> + TimePartitionUtils.getTimePartitionSlot( + time, dataPartitionQueryParam.getDatabaseName())) + .distinct() + .collect(Collectors.toList()) + : dataPartitionQueryParam.getTimePartitionSlotList(); + for (TTimePartitionSlot timePartitionSlot : timePartitionSlots) { if (timePartitionSlot.startTime == 0) { timePartitionSlotListMap.put(timePartitionSlot, d1DataRegions); } else { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java index aa31e0ed3762d..fc464048a06cc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java @@ -361,13 +361,15 @@ public void testGetTimePartitionSlotList() { // time >= 10 and time <= 9 Pair, Pair> res = getTimePartitionSlotList( - FilterFactory.and(TimeFilterApi.gtEq(10), TimeFilterApi.ltEq(9)), context); + FilterFactory.and(TimeFilterApi.gtEq(10), TimeFilterApi.ltEq(9)), + context, + context.getDatabase()); assertTrue(res.left.isEmpty()); assertFalse(res.right.left); assertFalse(res.right.right); // time >= 10 - res = getTimePartitionSlotList(TimeFilterApi.gtEq(10), context); + res = getTimePartitionSlotList(TimeFilterApi.gtEq(10), context, context.getDatabase()); assertEquals(1, res.left.size()); List expected = Collections.singletonList(new TTimePartitionSlot(0)); assertEquals(expected.size(), res.left.size()); @@ -378,7 +380,7 @@ public void testGetTimePartitionSlotList() { assertTrue(res.right.right); // time < 20 - res = getTimePartitionSlotList(TimeFilterApi.lt(20), context); + res = getTimePartitionSlotList(TimeFilterApi.lt(20), context, context.getDatabase()); assertEquals(1, res.left.size()); expected = Collections.singletonList(new TTimePartitionSlot(0)); assertEquals(expected.size(), res.left.size()); @@ -391,7 +393,9 @@ public void testGetTimePartitionSlotList() { // time > 10 and time <= 20 res = getTimePartitionSlotList( - FilterFactory.and(TimeFilterApi.gt(10), TimeFilterApi.ltEq(20)), context); + FilterFactory.and(TimeFilterApi.gt(10), TimeFilterApi.ltEq(20)), + context, + context.getDatabase()); expected = Collections.singletonList(new TTimePartitionSlot(0)); assertEquals(expected.size(), res.left.size()); for (int i = 0; i < expected.size(); i++) { @@ -408,7 +412,8 @@ public void testGetTimePartitionSlotList() { TimeFilterApi.gt(0), TimeFilterApi.ltEq( CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3 + 1)), - context); + context, + context.getDatabase()); expected = Arrays.asList( new TTimePartitionSlot(0), @@ -434,7 +439,8 @@ public void testGetTimePartitionSlotList() { CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1), TimeFilterApi.lt( CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)), - context); + context, + context.getDatabase()); expected = Arrays.asList( new TTimePartitionSlot(0), @@ -454,7 +460,8 @@ public void testGetTimePartitionSlotList() { TimeFilterApi.between( CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1, CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()), - context); + context, + context.getDatabase()); expected = Arrays.asList( new TTimePartitionSlot(0), @@ -476,7 +483,8 @@ public void testGetTimePartitionSlotList() { CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()), TimeFilterApi.ltEq( CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)), - context); + context, + context.getDatabase()); expected = Collections.singletonList( new TTimePartitionSlot( @@ -495,7 +503,8 @@ public void testGetTimePartitionSlotList() { TimeFilterApi.between( CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(), CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1), - context); + context, + context.getDatabase()); expected = Collections.singletonList( new TTimePartitionSlot( @@ -557,7 +566,7 @@ public void testGetTimePartitionSlotList() { CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() * 5 + 10))); - res = getTimePartitionSlotList(orFilter4, context); + res = getTimePartitionSlotList(orFilter4, context, context.getDatabase()); expected = Arrays.asList( new TTimePartitionSlot(0), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java index 9ca9cfb8bff2a..9c61de20d67db 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java @@ -25,10 +25,12 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -149,6 +151,7 @@ private static String getDeviceName(String storageGroupName, int deviceNumber) { @Before public void setUp() throws Exception { + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); partitionCache = new PartitionCache(); partitionCache.updateDatabaseCache(databases); partitionCache.updateSchemaPartitionCache(schemaPartitionTable); @@ -159,6 +162,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { partitionCache.invalidAllCache(); + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); } @Test @@ -268,6 +272,28 @@ public void testNeedLastCacheDefaultsToTrueWhenUnset() { assertTrue(partitionCache.isNeedLastCache("root.missing")); } + @Test + public void testTimePartitionConfigCache() { + final TDatabaseSchema databaseSchema = new TDatabaseSchema(getDatabaseName(0)); + databaseSchema.setTimePartitionOrigin(123L); + databaseSchema.setTimePartitionInterval(456L); + + partitionCache.updateDatabaseCache( + Collections.singletonMap(databaseSchema.getName(), databaseSchema)); + + assertEquals(123L, TimePartitionUtils.getTimePartitionOrigin(databaseSchema.getName())); + assertEquals(456L, TimePartitionUtils.getTimePartitionInterval(databaseSchema.getName())); + + partitionCache.removeFromDatabaseCache(); + + assertEquals( + CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(), + TimePartitionUtils.getTimePartitionOrigin(databaseSchema.getName())); + assertEquals( + CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(), + TimePartitionUtils.getTimePartitionInterval(databaseSchema.getName())); + } + @Test public void testRegionReplicaSetCache() { // test update regionReplicaSetCache with small timestamp diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTaskTest.java index 3f698dacddd12..181a3947bf435 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTaskTest.java @@ -48,14 +48,16 @@ public void testShowCreateDatabaseSQL() { final TDatabaseInfo databaseInfo = new TDatabaseInfo(); databaseInfo.setName("test_db"); databaseInfo.setTTL(Long.MAX_VALUE); + databaseInfo.setTimePartitionOrigin(100L); databaseInfo.setTimePartitionInterval(604800000L); databaseInfo.setMaxSchemaRegionNum(0); databaseInfo.setMaxDataRegionNum(1); + databaseInfo.setNeedLastCache(true); assertEquals( "CREATE DATABASE \"test_db\" WITH (ttl='" + IoTDBConstant.TTL_INFINITE - + "',time_partition_interval=604800000,max_schema_region_group_num=0,max_data_region_group_num=1)", + + "',time_partition_origin=100,time_partition_interval=604800000,max_schema_region_group_num=0,max_data_region_group_num=1,need_last_cache=true)", ShowCreateDatabaseTask.getShowCreateDatabaseSQL(databaseInfo)); } @@ -64,12 +66,14 @@ public void testShowCreateDatabaseSQLShouldQuoteIdentifierAndKeepFiniteTTL() { final TDatabaseInfo databaseInfo = new TDatabaseInfo(); databaseInfo.setName("test\"db"); databaseInfo.setTTL(123L); + databaseInfo.setTimePartitionOrigin(200L); databaseInfo.setTimePartitionInterval(1000L); databaseInfo.setMaxSchemaRegionNum(2); databaseInfo.setMaxDataRegionNum(3); + databaseInfo.setNeedLastCache(false); assertEquals( - "CREATE DATABASE \"test\"\"db\" WITH (ttl=123,time_partition_interval=1000,max_schema_region_group_num=2,max_data_region_group_num=3)", + "CREATE DATABASE \"test\"\"db\" WITH (ttl=123,time_partition_origin=200,time_partition_interval=1000,max_schema_region_group_num=2,max_data_region_group_num=3,need_last_cache=false)", ShowCreateDatabaseTask.getShowCreateDatabaseSQL(databaseInfo)); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java index f7da4e0614496..64424099fbbea 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java @@ -35,6 +35,7 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; @@ -85,6 +86,10 @@ public void setUp() { CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(0); CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(100); + TDatabaseSchema databaseSchema = new TDatabaseSchema(); + databaseSchema.setTimePartitionInterval(100); + TimePartitionUtils.updateDatabaseTimePartitionConfig("root.sg1", databaseSchema); + TimePartitionUtils.updateDatabaseTimePartitionConfig("root.sg2", databaseSchema); TimePartitionUtils.setTimePartitionOrigin(0); TimePartitionUtils.setTimePartitionInterval(100); @@ -116,13 +121,13 @@ private void initDataPartitionMap() { for (int i = 0; i < seriesSlotPartitionNum; i++) { Map> timePartitionSlotMap = new HashMap<>(); for (int t = -2; t < 5; t++) { - long startTime = t * TimePartitionUtils.getTimePartitionInterval() + 1; + long startTime = t * TimePartitionUtils.getTimePartitionInterval("root.sg1") + 1; timePartitionSlotMap.put( - TimePartitionUtils.getTimePartitionSlot(startTime), + TimePartitionUtils.getTimePartitionSlot(startTime, "root.sg1"), Collections.singletonList( new TRegionReplicaSet( new TConsensusGroupId( - TConsensusGroupType.DataRegion, getRegionIdByTime(startTime)), + TConsensusGroupType.DataRegion, getRegionIdByTime(startTime, "root.sg1")), locationList))); } @@ -137,7 +142,7 @@ TConsensusGroupType.DataRegion, getRegionIdByTime(startTime)), Map> timePartitionSlotMap = new HashMap<>(); for (int t = 0; t < 5; t++) { timePartitionSlotMap.put( - new TTimePartitionSlot(t * TimePartitionUtils.getTimePartitionInterval()), + new TTimePartitionSlot(t * TimePartitionUtils.getTimePartitionInterval("root.sg2")), Collections.singletonList( new TRegionReplicaSet( new TConsensusGroupId(TConsensusGroupType.DataRegion, 99), locationList))); @@ -161,8 +166,14 @@ private void initSchemaPartitionMap() { schemaPartitionMap.put("root.sg1", seriesPartitionSlotMap); } - private int getRegionIdByTime(long startTime) { - return (int) (4 - ((startTime - 1) / TimePartitionUtils.getTimePartitionInterval())); + private int getRegionIdByTime(long startTime, String database) { + long interval = TimePartitionUtils.getTimePartitionInterval(database); + return (int) (4 - ((startTime - 1) / interval)); + } + + private String getDatabaseName(PartialPath devicePath) { + String fullPath = devicePath.getFullPath(); + return fullPath.substring(0, fullPath.indexOf(".", fullPath.indexOf(".") + 1)); } protected DataPartition getDataPartition( @@ -219,7 +230,8 @@ public void testSplitInsertTablet() throws IllegalPathException { DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID( insertTabletNode.getTargetPath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(insertTabletNode.getTimePartitionSlots()); + dataPartitionQueryParam.setTimePartitionSlotList( + insertTabletNode.getTimePartitionSlots("root.sg1")); DataPartition dataPartition = getDataPartition(Collections.singletonList(dataPartitionQueryParam)); @@ -233,7 +245,7 @@ public void testSplitInsertTablet() throws IllegalPathException { InsertTabletNode tabletNode = (InsertTabletNode) insertNode; Assert.assertEquals(tabletNode.getTimes().length, 2); TConsensusGroupId regionId = tabletNode.getDataRegionReplicaSet().getRegionId(); - Assert.assertEquals(getRegionIdByTime(tabletNode.getMinTime()), regionId.getId()); + Assert.assertEquals(getRegionIdByTime(tabletNode.getMinTime(), "root.sg1"), regionId.getId()); if (tabletNode.getTimes()[0] == 1) { Assert.assertNotNull(tabletNode.getBitMaps()); Assert.assertTrue(tabletNode.getBitMaps()[0].isMarked(0)); @@ -252,7 +264,8 @@ public void testSplitInsertTablet() throws IllegalPathException { dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID( insertTabletNode.getTargetPath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(insertTabletNode.getTimePartitionSlots()); + dataPartitionQueryParam.setTimePartitionSlotList( + insertTabletNode.getTimePartitionSlots("root.sg2")); dataPartition = getDataPartition(Collections.singletonList(dataPartitionQueryParam)); analysis = new Analysis(); @@ -270,7 +283,7 @@ public void testSplitInsertTablet() throws IllegalPathException { public void testInsertTabletDoesNotSplitAtLongMaxValueWithinLastPartition() throws IllegalPathException { final long lastPartitionStartTime = - TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).getStartTime(); + TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE, "root.sg1").getStartTime(); final List expectedTimePartitionSlots = Collections.singletonList(new TTimePartitionSlot(lastPartitionStartTime)); @@ -278,18 +291,22 @@ public void testInsertTabletDoesNotSplitAtLongMaxValueWithinLastPartition() insertTabletNode.setTargetPath(new PartialPath("root.sg1.d1")); insertTabletNode.setTimes(new long[] {lastPartitionStartTime, Long.MAX_VALUE}); - Assert.assertEquals(expectedTimePartitionSlots, insertTabletNode.getTimePartitionSlots()); + Assert.assertEquals( + expectedTimePartitionSlots, insertTabletNode.getTimePartitionSlots("root.sg1")); InsertTabletStatement insertTabletStatement = new InsertTabletStatement(); insertTabletStatement.setTimes(new long[] {lastPartitionStartTime, Long.MAX_VALUE}); - Assert.assertEquals(expectedTimePartitionSlots, insertTabletStatement.getTimePartitionSlots()); + Assert.assertEquals( + expectedTimePartitionSlots, insertTabletStatement.getTimePartitionSlots("root.sg1")); } @Test public void testInsertTabletSplitsAtExactLongMaxValueUpperBound() throws IllegalPathException { CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(1); TimePartitionUtils.setTimePartitionInterval(1); + TimePartitionUtils.updateDatabaseTimePartitionConfig( + "root.sg1", new TDatabaseSchema().setTimePartitionInterval(1).setTimePartitionOrigin(0)); final List expectedTimePartitionSlots = Arrays.asList( new TTimePartitionSlot(Long.MAX_VALUE - 1), new TTimePartitionSlot(Long.MAX_VALUE)); @@ -299,12 +316,14 @@ public void testInsertTabletSplitsAtExactLongMaxValueUpperBound() throws Illegal insertTabletNode.setTargetPath(new PartialPath("root.sg1.d1")); insertTabletNode.setTimes(new long[] {Long.MAX_VALUE - 1, Long.MAX_VALUE}); - Assert.assertEquals(expectedTimePartitionSlots, insertTabletNode.getTimePartitionSlots()); + Assert.assertEquals( + expectedTimePartitionSlots, insertTabletNode.getTimePartitionSlots("root.sg1")); InsertTabletStatement insertTabletStatement = new InsertTabletStatement(); insertTabletStatement.setTimes(new long[] {Long.MAX_VALUE - 1, Long.MAX_VALUE}); - Assert.assertEquals(expectedTimePartitionSlots, insertTabletStatement.getTimePartitionSlots()); + Assert.assertEquals( + expectedTimePartitionSlots, insertTabletStatement.getTimePartitionSlots("root.sg1")); } @Test @@ -330,7 +349,8 @@ public void testSplitInsertTabletSkipsClearedMeasurementWithRetainedColumn() DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID( insertTabletNode.getTargetPath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(insertTabletNode.getTimePartitionSlots()); + dataPartitionQueryParam.setTimePartitionSlotList( + insertTabletNode.getTimePartitionSlots("root.sg1")); DataPartition dataPartition = getDataPartition(Collections.singletonList(dataPartitionQueryParam)); @@ -377,13 +397,13 @@ public void testSplitRelationalInsertTablet() throws IllegalPathException { DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID(relationalInsertTabletNode.getDeviceID(0)); dataPartitionQueryParam.setTimePartitionSlotList( - relationalInsertTabletNode.getTimePartitionSlots()); + relationalInsertTabletNode.getTimePartitionSlots("root.sg1")); dataPartitionQueryParamList.add(dataPartitionQueryParam); dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID(relationalInsertTabletNode.getDeviceID(1)); dataPartitionQueryParam.setTimePartitionSlotList( - relationalInsertTabletNode.getTimePartitionSlots()); + relationalInsertTabletNode.getTimePartitionSlots("root.sg1")); dataPartitionQueryParamList.add(dataPartitionQueryParam); DataPartition dataPartition = getDataPartition(dataPartitionQueryParamList); @@ -400,7 +420,7 @@ public void testSplitRelationalInsertTablet() throws IllegalPathException { // keep the time order after split Assert.assertTrue(tabletNode.getTimes()[0] < tabletNode.getTimes()[1]); TConsensusGroupId regionId = tabletNode.getDataRegionReplicaSet().getRegionId(); - Assert.assertEquals(getRegionIdByTime(tabletNode.getMinTime()), regionId.getId()); + Assert.assertEquals(getRegionIdByTime(tabletNode.getMinTime(), "root.sg1"), regionId.getId()); if (tabletNode.getTimes()[0] == 1) { Assert.assertNotNull(tabletNode.getBitMaps()); Assert.assertNull(tabletNode.getBitMaps()[0]); @@ -441,7 +461,9 @@ public void testInsertMultiTablets() throws IllegalPathException { DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID( insertTabletNode.getTargetPath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(insertTabletNode.getTimePartitionSlots()); + dataPartitionQueryParam.setTimePartitionSlotList( + insertTabletNode.getTimePartitionSlots( + getDatabaseName(insertTabletNode.getTargetPath()))); dataPartitionQueryParams.add(dataPartitionQueryParam); } @@ -456,13 +478,18 @@ public void testInsertMultiTablets() throws IllegalPathException { @Test public void testInsertRowsNode() throws IllegalPathException { + TDatabaseSchema databaseSchema = new TDatabaseSchema(); + databaseSchema.setTimePartitionInterval(150); + TimePartitionUtils.updateDatabaseTimePartitionConfig("root.sg2", databaseSchema); + initDataPartitionMap(); InsertRowsNode insertRowsNode = new InsertRowsNode(new PlanNodeId("plan node 3")); for (int i = 0; i < 7; i++) { InsertRowNode insertRowNode = new InsertRowNode(new PlanNodeId("plan node 3")); insertRowNode.setTargetPath(new PartialPath(String.format("root.sg1.d%d", i))); - insertRowNode.setTime((i - 2) * TimePartitionUtils.getTimePartitionInterval()); + long interval = TimePartitionUtils.getTimePartitionInterval("root.sg1"); + insertRowNode.setTime((i - 2) * interval); insertRowsNode.addOneInsertRowNode(insertRowNode, 2 * i); insertRowNode = new InsertRowNode(new PlanNodeId("plan node 3")); @@ -475,7 +502,8 @@ public void testInsertRowsNode() throws IllegalPathException { for (InsertRowNode insertRowNode : insertRowsNode.getInsertRowNodeList()) { DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID(insertRowNode.getTargetPath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(insertRowNode.getTimePartitionSlots()); + dataPartitionQueryParam.setTimePartitionSlotList( + insertRowNode.getTimePartitionSlots(getDatabaseName(insertRowNode.getTargetPath()))); dataPartitionQueryParams.add(dataPartitionQueryParam); } @@ -514,7 +542,8 @@ public void testInsertRowsOfOneDeviceNode() throws IllegalPathException { for (InsertRowNode insertRowNode : insertRowsOfOneDeviceNode.getInsertRowNodeList()) { DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID(insertRowNode.getTargetPath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(insertRowNode.getTimePartitionSlots()); + dataPartitionQueryParam.setTimePartitionSlotList( + insertRowNode.getTimePartitionSlots("root.sg1")); dataPartitionQueryParams.add(dataPartitionQueryParam); } @@ -551,7 +580,8 @@ public void testInsertRowsOfOneDeviceNode() throws IllegalPathException { for (InsertRowNode insertRowNode : insertRowsOfOneDeviceNode.getInsertRowNodeList()) { DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDeviceID(insertRowNode.getTargetPath().getIDeviceIDAsFullDevice()); - dataPartitionQueryParam.setTimePartitionSlotList(insertRowNode.getTimePartitionSlots()); + dataPartitionQueryParam.setTimePartitionSlotList( + insertRowNode.getTimePartitionSlots("root.sg2")); dataPartitionQueryParams.add(dataPartitionQueryParam); } @@ -570,6 +600,10 @@ public void testInsertRowsOfOneDeviceNode() throws IllegalPathException { @After public void tearDown() { + TDatabaseSchema databaseSchema = new TDatabaseSchema(); + databaseSchema.setTimePartitionInterval(prevTimePartitionInterval); + TimePartitionUtils.updateDatabaseTimePartitionConfig("root.sg1", databaseSchema); + TimePartitionUtils.updateDatabaseTimePartitionConfig("root.sg2", databaseSchema); TimePartitionUtils.setTimePartitionInterval(prevTimePartitionInterval); TimePartitionUtils.setTimePartitionOrigin(prevTimePartitionOrigin); CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(prevTimePartitionInterval); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java index 9c9918ddbc8c4..7679381ffaabb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java @@ -59,6 +59,7 @@ import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.session.InternalClientSession; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -1444,8 +1445,14 @@ public DataPartition getOrCreateDataPartition( final TSeriesPartitionSlot partitionSlot = seriesPartitionExecutor.getSeriesPartitionSlot(dataPartitionQueryParam.getDeviceID()); - for (final TTimePartitionSlot tTimePartitionSlot : - dataPartitionQueryParam.getTimePartitionSlotList()) { + final List timePartitionSlots = + dataPartitionQueryParam.hasTimeList() + ? dataPartitionQueryParam.getTimeList().stream() + .map(time -> TimePartitionUtils.getTimePartitionSlot(time, databaseName)) + .distinct() + .collect(Collectors.toList()) + : dataPartitionQueryParam.getTimePartitionSlotList(); + for (final TTimePartitionSlot tTimePartitionSlot : timePartitionSlots) { dataPartitionMap .computeIfAbsent(databaseName, d -> new HashMap<>()) .computeIfAbsent(partitionSlot, slot -> new HashMap<>()) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/StorageEngineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/StorageEngineTest.java index b0b9ce2880ab8..95b05b6b4f81a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/StorageEngineTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/StorageEngineTest.java @@ -78,15 +78,17 @@ public void testGetAllDataRegionIds() throws Exception { @Test public void testGetTimePartitionId() { - long timePartitionInterval = TimePartitionUtils.getTimePartitionInterval(); - Assert.assertEquals(-2, TimePartitionUtils.getTimePartitionId(-timePartitionInterval - 1)); - Assert.assertEquals(-1, TimePartitionUtils.getTimePartitionId(-timePartitionInterval)); - Assert.assertEquals(-1, TimePartitionUtils.getTimePartitionId(-1)); - Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(0)); - Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(1)); - Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(timePartitionInterval / 2)); - Assert.assertEquals(1, TimePartitionUtils.getTimePartitionId(timePartitionInterval * 2 - 1)); - Assert.assertEquals(2, TimePartitionUtils.getTimePartitionId(timePartitionInterval * 2 + 1)); + long timePartitionInterval = TimePartitionUtils.getTimePartitionInterval("root.db"); + long interval = TimePartitionUtils.getTimePartitionInterval("root.db"); + Assert.assertEquals(-2, TimePartitionUtils.getTimePartitionId(-interval - 1, "root.db")); + Assert.assertEquals(-1, TimePartitionUtils.getTimePartitionId(-interval, "root.db")); + Assert.assertEquals(-1, TimePartitionUtils.getTimePartitionId(-1, "root.db")); + Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(0, "root.db")); + Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(1, "root.db")); + Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(interval / 2, "root.db")); + Assert.assertEquals(1, TimePartitionUtils.getTimePartitionId(interval * 2 - 1, "root.db")); + Assert.assertEquals( + 2, TimePartitionUtils.getTimePartitionId(timePartitionInterval * 2 + 1, "root.db")); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index f7a594659e771..10599507aaf6e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -143,6 +143,7 @@ public class DataRegionTest { private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); private static final Logger logger = LoggerFactory.getLogger(DataRegionTest.class); + private String databaseName = "root.vehicle"; private String storageGroup = "root.vehicle.d0"; private String systemDir = TestConstant.OUTPUT_DATA_DIR.concat("info"); private String deviceId = "root.vehicle.d0"; @@ -1852,7 +1853,8 @@ public void testDeleteDataNotInFlushingMemtable() record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } - TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + TsFileResource tsFileResource = + dataRegion.getTsFileManager().getTsFileList(true, databaseName).get(0); TsFileProcessor tsFileProcessor = tsFileResource.getProcessor(); tsFileProcessor.getFlushingMemTable().addLast(tsFileProcessor.getWorkMemTable()); @@ -1882,7 +1884,8 @@ public void testDeleteDataInSeqFlushingMemtable() record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } - TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + TsFileResource tsFileResource = + dataRegion.getTsFileManager().getTsFileList(true, databaseName).get(0); TsFileProcessor tsFileProcessor = tsFileResource.getProcessor(); tsFileProcessor.getFlushingMemTable().addLast(tsFileProcessor.getWorkMemTable()); @@ -1925,7 +1928,8 @@ public void testDeleteDataInUnSeqFlushingMemtable() record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } - TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + TsFileResource tsFileResource = + dataRegion.getTsFileManager().getTsFileList(true, databaseName).get(0); MeasurementPath path = new MeasurementPath("root.vehicle.d0.s0"); DeleteDataNode deleteDataNode1 = @@ -1978,7 +1982,7 @@ public void testDeleteDataInUnSeqFlushingMemtable() Assert.assertFalse(tsFileResource.anyModFileExists()); - tsFileResource = dataRegion.getTsFileManager().getTsFileList(false).get(0); + tsFileResource = dataRegion.getTsFileManager().getTsFileList(false, databaseName).get(0); TsFileProcessor tsFileProcessor = tsFileResource.getProcessor(); tsFileProcessor.getFlushingMemTable().addLast(tsFileProcessor.getWorkMemTable()); @@ -2025,7 +2029,8 @@ public void testDeleteDataInSeqWorkingMemtable() record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } - TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + TsFileResource tsFileResource = + dataRegion.getTsFileManager().getTsFileList(true, databaseName).get(0); MeasurementPath path = new MeasurementPath("root.vehicle.d0.s0"); DeleteDataNode deleteDataNode1 = @@ -2062,7 +2067,8 @@ public void testFlushingEmptyMemtable() record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } - TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + TsFileResource tsFileResource = + dataRegion.getTsFileManager().getTsFileList(true, databaseName).get(0); MeasurementPath path = new MeasurementPath("root.vehicle.d0.s0"); DeleteDataNode deleteDataNode = @@ -2147,7 +2153,8 @@ public void testDeleteDataDirectlySeqWriteModsOrDeleteFiles() dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } - TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + TsFileResource tsFileResource = + dataRegion.getTsFileManager().getTsFileList(true, databaseName).get(0); // delete data in work mem, no mods. MeasurementPath path = new MeasurementPath("root.vehicle.d0.**"); DeleteDataNode deleteDataNode1 = @@ -2184,7 +2191,8 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles() record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } - TsFileResource tsFileResourceSeq = dataRegion.getTsFileManager().getTsFileList(true).get(0); + TsFileResource tsFileResourceSeq = + dataRegion.getTsFileManager().getTsFileList(true, databaseName).get(0); dataRegion.syncCloseAllWorkingTsFileProcessors(); for (int j = 30; j < 100; j++) { TSRecord record = new TSRecord(deviceId, j); @@ -2193,7 +2201,8 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles() } dataRegion.syncCloseWorkingTsFileProcessors(true); - TsFileResource tsFileResourceUnSeq = dataRegion.getTsFileManager().getTsFileList(false).get(0); + TsFileResource tsFileResourceUnSeq = + dataRegion.getTsFileManager().getTsFileList(false, databaseName).get(0); assertTrue(tsFileResourceSeq.getTsFile().exists()); assertTrue(tsFileResourceUnSeq.getTsFile().exists()); @@ -2433,7 +2442,8 @@ public void testFlushSpecifiedResource() record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } - TsFileResource tsFileResourceSeq = dataRegion.getTsFileManager().getTsFileList(true).get(0); + TsFileResource tsFileResourceSeq = + dataRegion.getTsFileManager().getTsFileList(true, databaseName).get(0); Future future = dataRegion.asyncCloseOneTsFileProcessor(tsFileResourceSeq); Future future2 = dataRegion.asyncCloseOneTsFileProcessor(tsFileResourceSeq); assertTrue(future == future2 || future2 instanceof CompletableFuture); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java index 06ec607a704e8..390074b93ec66 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java @@ -385,8 +385,8 @@ public void testTTLRemoval() dataRegion.getTsFileManager(), timePartition, new CompactionScheduleContext(), true); } long totalWaitingTime = 0; - while (dataRegion.getTsFileManager().getTsFileList(true).size() - + dataRegion.getTsFileManager().getTsFileList(false).size() + while (dataRegion.getTsFileManager().getTsFileList(true, "root.TTL_SG1").size() + + dataRegion.getTsFileManager().getTsFileList(false, "root.TTL_SG1").size() != 0) { sleep(200); totalWaitingTime += 200; @@ -476,8 +476,8 @@ public void testTTLCleanFile() dataRegion.getTsFileManager(), timePartition, new CompactionScheduleContext(), true); } long totalWaitingTime = 0; - while (dataRegion.getTsFileManager().getTsFileList(true).size() - + dataRegion.getTsFileManager().getTsFileList(false).size() + while (dataRegion.getTsFileManager().getTsFileList(true, "root.TTL_SG1").size() + + dataRegion.getTsFileManager().getTsFileList(false, "root.TTL_SG1").size() != 0) { sleep(200); totalWaitingTime += 200; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TsFileManagerTest.java index 07989246ae4d4..6983300086b32 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TsFileManagerTest.java @@ -49,6 +49,7 @@ public class TsFileManagerTest { private TsFileManager tsFileManager; private List seqResources; private List unseqResources; + private static final String databaseName = "test"; @Before public void setUp() throws IOException, WriteProcessException, MetadataException { @@ -87,8 +88,8 @@ public void testAddRemoveAndIterator() { tsFileManager.add(tsFileResource, true); } tsFileManager.addAll(unseqResources, false); - assertEquals(5, tsFileManager.getTsFileList(true).size()); - assertEquals(4, tsFileManager.getTsFileList(false).size()); + assertEquals(5, tsFileManager.getTsFileList(true, databaseName).size()); + assertEquals(4, tsFileManager.getTsFileList(false, databaseName).size()); assertEquals(5, tsFileManager.size(true)); assertEquals(4, tsFileManager.size(false)); assertTrue(tsFileManager.contains(seqResources.get(0), true)); @@ -124,20 +125,20 @@ public void testAddRemoveAndIterator() { false)); assertFalse(tsFileManager.isEmpty(true)); assertFalse(tsFileManager.isEmpty(false)); - tsFileManager.remove(tsFileManager.getTsFileList(true).get(0), true); - tsFileManager.remove(tsFileManager.getTsFileList(false).get(0), false); - assertEquals(4, tsFileManager.getTsFileList(true).size()); - tsFileManager.removeAll(tsFileManager.getTsFileList(false), false); - assertEquals(0, tsFileManager.getTsFileList(false).size()); + tsFileManager.remove(tsFileManager.getTsFileList(true, databaseName).get(0), true); + tsFileManager.remove(tsFileManager.getTsFileList(false, databaseName).get(0), false); + assertEquals(4, tsFileManager.getTsFileList(true, databaseName).size()); + tsFileManager.removeAll(tsFileManager.getTsFileList(false, databaseName), false); + assertEquals(0, tsFileManager.getTsFileList(false, databaseName).size()); long count = 0; - Iterator iterator = tsFileManager.getIterator(true); + Iterator iterator = tsFileManager.getIterator(true, databaseName); while (iterator.hasNext()) { iterator.next(); count++; } assertEquals(4, count); - tsFileManager.removeAll(tsFileManager.getTsFileList(true), true); - assertEquals(0, tsFileManager.getTsFileList(true).size()); + tsFileManager.removeAll(tsFileManager.getTsFileList(true, databaseName), true); + assertEquals(0, tsFileManager.getTsFileList(true, databaseName).size()); assertTrue(tsFileManager.isEmpty(true)); assertTrue(tsFileManager.isEmpty(false)); tsFileManager.add( @@ -181,16 +182,16 @@ public void testIteratorRemove() { tsFileManager.add(tsFileResource, true); } tsFileManager.addAll(seqResources, false); - assertEquals(5, tsFileManager.getTsFileList(true).size()); + assertEquals(5, tsFileManager.getTsFileList(true, databaseName).size()); - Iterator tsFileResourceIterator = tsFileManager.getIterator(true); + Iterator tsFileResourceIterator = tsFileManager.getIterator(true, databaseName); tsFileResourceIterator.next(); try { tsFileResourceIterator.remove(); } catch (UnsupportedOperationException e) { // pass } - assertEquals(5, tsFileManager.getTsFileList(true).size()); + assertEquals(5, tsFileManager.getTsFileList(true, databaseName).size()); TsFileResource tsFileResource1 = new TsFileResource( @@ -232,7 +233,8 @@ public void testIteratorRemove() { + 0 + ".tsfile"))); tsFileManager.add(tsFileResource3, true); - Iterator tsFileResourceIterator2 = tsFileManager.getIterator(true); + Iterator tsFileResourceIterator2 = + tsFileManager.getIterator(true, databaseName); int count = 0; while (tsFileResourceIterator2.hasNext()) { count++; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java index b51910f84efdc..1face79d4e1c2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java @@ -562,7 +562,7 @@ private void removeFiles() throws IOException { protected void validateSeqFiles(boolean isSeq) { TsFileValidationTool.clearMap(true); List files = new ArrayList<>(); - for (TsFileResource resource : tsFileManager.getTsFileList(isSeq)) { + for (TsFileResource resource : tsFileManager.getTsFileList(isSeq, COMPACTION_TEST_SG)) { files.add(resource.getTsFile()); } TsFileValidationTool.findIncorrectFiles(files); @@ -580,8 +580,8 @@ protected Map> readSourceFiles( path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); while (tsBlockReader.hasNextBatch()) { TsBlock block = tsBlockReader.nextBatch(); @@ -607,8 +607,8 @@ protected void validateTargetDatas( entry.getKey(), FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); List timeseriesData = entry.getValue(); tmpSourceDatas.put(entry.getKey(), new ArrayList<>(timeseriesData)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java index 5c2bf1c8e4322..f7ca4351a7e20 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java @@ -390,7 +390,7 @@ private void consumeChunkDataAndValidate(TsFileResource resource) } return true; }); - splitter.splitTsFileByDataPartition(); + splitter.splitTsFileByDataPartition(resource.getDatabaseName()); List splitResources = new ArrayList<>(); for (Map.Entry entry : writerMap.entrySet()) { TestLoadTsFileIOWriter writer = entry.getValue(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerTest.java index b1377dbbba6bb..a6274abceccba 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerTest.java @@ -188,7 +188,7 @@ public void test1() throws IOException, MetadataException, InterruptedException long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 1) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -204,7 +204,7 @@ public void test1() throws IOException, MetadataException, InterruptedException totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 1) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -220,7 +220,7 @@ public void test1() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -313,7 +313,7 @@ public void test2() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 1) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -328,7 +328,7 @@ public void test2() throws IOException, MetadataException, InterruptedException } CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(10); totalWaitingTime += 10; @@ -339,8 +339,8 @@ public void test2() throws IOException, MetadataException, InterruptedException if (totalWaitingTime % 10_000 == 0) { logger.warn( "sequence file num is {}, unsequence file num is {}", - tsFileManager.getTsFileList(true).size(), - tsFileManager.getTsFileList(false).size()); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size(), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } if (totalWaitingTime % SCHEDULE_AGAIN_TIME == 0) { logger.warn("Has waited for {} s, Schedule again", totalWaitingTime / 1000); @@ -350,7 +350,7 @@ public void test2() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - // assertEquals(100, tsFileManager.getTsFileList(true).size()); + // assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -423,7 +423,7 @@ public void test3() throws IOException, MetadataException, InterruptedException } long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 1) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -438,7 +438,7 @@ public void test3() throws IOException, MetadataException, InterruptedException } } totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -448,7 +448,7 @@ public void test3() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(1, tsFileManager.getTsFileList(true).size()); + assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -520,7 +520,7 @@ public void test4() throws IOException, MetadataException, InterruptedException tsFileManager.add(tsFileResource, false); } long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -534,7 +534,7 @@ public void test4() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -605,7 +605,7 @@ public void test5() throws IOException, MetadataException, InterruptedException } long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 1) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -620,7 +620,7 @@ public void test5() throws IOException, MetadataException, InterruptedException } } totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 1) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -636,7 +636,7 @@ public void test5() throws IOException, MetadataException, InterruptedException } totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -719,12 +719,12 @@ public void test6() throws IOException, MetadataException, InterruptedException tsFileManager.add(tsFileResource, false); } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 1) { - assertEquals(100, tsFileManager.getTsFileList(true).size()); + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 1) { + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); try { Thread.sleep(100); totalWaitingTime += 100; @@ -741,8 +741,8 @@ public void test6() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { - assertEquals(100, tsFileManager.getTsFileList(true).size()); + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); try { Thread.sleep(100); totalWaitingTime += 100; @@ -756,7 +756,7 @@ public void test6() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -828,7 +828,7 @@ public void test7() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 1) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -845,7 +845,7 @@ public void test7() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -932,7 +932,7 @@ public void test8() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -946,7 +946,7 @@ public void test8() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1020,7 +1020,7 @@ public void test9() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 50) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 50) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1037,7 +1037,7 @@ public void test9() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 25) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 25) { Thread.sleep(100); totalWaitingTime += 100; CompactionScheduler.scheduleCompaction(tsFileManager, 0); @@ -1046,7 +1046,7 @@ public void test9() throws IOException, MetadataException, InterruptedException fail(); } } - assertTrue(tsFileManager.getTsFileList(true).size() <= 25); + assertTrue(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() <= 25); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1124,7 +1124,7 @@ public void test10() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 50) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 50) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1138,11 +1138,11 @@ public void test10() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 25) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 25) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1232,7 +1232,7 @@ public void test11() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 50) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 50) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1249,7 +1249,7 @@ public void test11() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 25) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 25) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1336,7 +1336,7 @@ public void test12() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 98) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 98) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1353,11 +1353,11 @@ public void test12() throws IOException, MetadataException, InterruptedException fail(e.getMessage()); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 96) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 96) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1371,7 +1371,7 @@ public void test12() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1442,16 +1442,16 @@ public void test14() throws IOException, MetadataException, InterruptedException tsFileManager.add(tsFileResource, false); } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 99) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 99) { try { Thread.sleep(100); totalWaitingTime += 100; - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); if (totalWaitingTime > MAX_WAITING_TIME) { @@ -1462,14 +1462,14 @@ public void test14() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 98) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 98) { try { Thread.sleep(100); - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); totalWaitingTime += 100; CompactionScheduler.scheduleCompaction(tsFileManager, 0); @@ -1481,7 +1481,7 @@ public void test14() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1555,7 +1555,7 @@ public void test15() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 99) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 99) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1572,7 +1572,7 @@ public void test15() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 98) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 98) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1586,7 +1586,7 @@ public void test15() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1672,7 +1672,7 @@ public void test16() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 98) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 98) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1686,11 +1686,11 @@ public void test16() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 96) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 96) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1704,7 +1704,7 @@ public void test16() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1804,10 +1804,10 @@ public void testLargeFileInLowerLevel() throws Exception { CompactionScheduler.scheduleCompaction(tsFileManager, 0); Thread.sleep(100); long sleepTime = 0; - while (tsFileManager.getTsFileList(true).size() >= 2) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() >= 2) { CompactionScheduler.scheduleCompaction(tsFileManager, 0); tsFileManager.readLock(); - List resources = tsFileManager.getTsFileList(true); + List resources = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); int previousFileLevel = TsFileNameGenerator.getTsFileName(resources.get(0).getTsFile().getName()) .getInnerCompactionCnt(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerWithFastPerformerTest.java index 747b2a67ea915..5c71a12858637 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerWithFastPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerWithFastPerformerTest.java @@ -185,7 +185,7 @@ public void test1() throws IOException, MetadataException, InterruptedException long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 1) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -201,7 +201,7 @@ public void test1() throws IOException, MetadataException, InterruptedException totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 1) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -217,7 +217,7 @@ public void test1() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -310,7 +310,7 @@ public void test2() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 1) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -325,7 +325,7 @@ public void test2() throws IOException, MetadataException, InterruptedException } CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(10); totalWaitingTime += 10; @@ -336,8 +336,8 @@ public void test2() throws IOException, MetadataException, InterruptedException if (totalWaitingTime % 10_000 == 0) { logger.warn( "sequence file num is {}, unsequence file num is {}", - tsFileManager.getTsFileList(true).size(), - tsFileManager.getTsFileList(false).size()); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size(), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } if (totalWaitingTime % SCHEDULE_AGAIN_TIME == 0) { logger.warn("Has waited for {} s, Schedule again", totalWaitingTime / 1000); @@ -347,7 +347,7 @@ public void test2() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - // assertEquals(100, tsFileManager.getTsFileList(true).size()); + // assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -420,7 +420,7 @@ public void test3() throws IOException, MetadataException, InterruptedException } long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 1) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -435,7 +435,7 @@ public void test3() throws IOException, MetadataException, InterruptedException } } totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -445,7 +445,7 @@ public void test3() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(1, tsFileManager.getTsFileList(true).size()); + assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -517,7 +517,7 @@ public void test4() throws IOException, MetadataException, InterruptedException tsFileManager.add(tsFileResource, false); } long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -531,7 +531,7 @@ public void test4() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -602,7 +602,7 @@ public void test5() throws IOException, MetadataException, InterruptedException } long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 1) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -617,7 +617,7 @@ public void test5() throws IOException, MetadataException, InterruptedException } } totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 1) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -633,7 +633,7 @@ public void test5() throws IOException, MetadataException, InterruptedException } totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -716,12 +716,12 @@ public void test6() throws IOException, MetadataException, InterruptedException tsFileManager.add(tsFileResource, false); } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 1) { - assertEquals(100, tsFileManager.getTsFileList(true).size()); + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 1) { + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); try { Thread.sleep(100); totalWaitingTime += 100; @@ -738,8 +738,8 @@ public void test6() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { - assertEquals(100, tsFileManager.getTsFileList(true).size()); + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); try { Thread.sleep(100); totalWaitingTime += 100; @@ -753,7 +753,7 @@ public void test6() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -825,7 +825,7 @@ public void test7() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 1) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 1) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -842,7 +842,7 @@ public void test7() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -929,7 +929,7 @@ public void test8() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -943,7 +943,7 @@ public void test8() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1017,7 +1017,7 @@ public void test9() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 50) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 50) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1034,7 +1034,7 @@ public void test9() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 25) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 25) { Thread.sleep(100); totalWaitingTime += 100; CompactionScheduler.scheduleCompaction(tsFileManager, 0); @@ -1043,7 +1043,7 @@ public void test9() throws IOException, MetadataException, InterruptedException fail(); } } - assertTrue(tsFileManager.getTsFileList(true).size() <= 25); + assertTrue(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() <= 25); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1121,7 +1121,7 @@ public void test10() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 50) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 50) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1135,11 +1135,11 @@ public void test10() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 25) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 25) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1229,7 +1229,7 @@ public void test11() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 50) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 50) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1246,7 +1246,7 @@ public void test11() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 25) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 25) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1333,7 +1333,7 @@ public void test12() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 98) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 98) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1347,11 +1347,11 @@ public void test12() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 96) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 96) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1365,7 +1365,7 @@ public void test12() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1436,16 +1436,16 @@ public void test14() throws IOException, MetadataException, InterruptedException tsFileManager.add(tsFileResource, false); } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 99) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 99) { try { Thread.sleep(100); totalWaitingTime += 100; - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); if (totalWaitingTime > MAX_WAITING_TIME) { @@ -1456,14 +1456,14 @@ public void test14() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 98) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 98) { try { Thread.sleep(100); - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); totalWaitingTime += 100; CompactionScheduler.scheduleCompaction(tsFileManager, 0); @@ -1475,7 +1475,7 @@ public void test14() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1549,7 +1549,7 @@ public void test15() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 99) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 99) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1566,7 +1566,7 @@ public void test15() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(true).size() > 98) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() > 98) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1580,7 +1580,7 @@ public void test15() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1666,7 +1666,7 @@ public void test16() throws IOException, MetadataException, InterruptedException CompactionScheduler.scheduleCompaction(tsFileManager, 0); long totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 98) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 98) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1680,11 +1680,11 @@ public void test16() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); CompactionScheduler.scheduleCompaction(tsFileManager, 0); totalWaitingTime = 0; - while (tsFileManager.getTsFileList(false).size() > 96) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 96) { try { Thread.sleep(100); totalWaitingTime += 100; @@ -1698,7 +1698,7 @@ public void test16() throws IOException, MetadataException, InterruptedException e.printStackTrace(); } } - assertEquals(100, tsFileManager.getTsFileList(true).size()); + assertEquals(100, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); tsFileManager.setAllowCompaction(false); stopCompactionTaskManager(); } finally { @@ -1798,10 +1798,10 @@ public void testLargeFileInLowerLevel() throws Exception { CompactionScheduler.scheduleCompaction(tsFileManager, 0); Thread.sleep(100); long sleepTime = 0; - while (tsFileManager.getTsFileList(true).size() >= 2) { + while (tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size() >= 2) { CompactionScheduler.scheduleCompaction(tsFileManager, 0); tsFileManager.readLock(); - List resources = tsFileManager.getTsFileList(true); + List resources = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); int previousFileLevel = TsFileNameGenerator.getTsFileName(resources.get(0).getTsFile().getName()) .getInnerCompactionCnt(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionWithMinTimestampTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionWithMinTimestampTest.java index 567c67128905a..91174a22dfd56 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionWithMinTimestampTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionWithMinTimestampTest.java @@ -119,7 +119,7 @@ public void testCompactionPerformer() throws IOException { new InnerSpaceCompactionTask( 0, tsFileManager, Collections.singletonList(resource), false, performer, 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(false).get(0); + TsFileResource target = tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0); Assert.assertEquals( Long.MIN_VALUE, target diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastAlignedCrossCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastAlignedCrossCompactionTest.java index f4c59e8743d51..b9c57d5d2dc99 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastAlignedCrossCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastAlignedCrossCompactionTest.java @@ -301,7 +301,7 @@ public void test1() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -606,7 +606,7 @@ public void test2() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -910,7 +910,7 @@ public void test3() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -1231,7 +1231,7 @@ public void test4() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -1667,7 +1667,7 @@ public void test5() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -2111,7 +2111,7 @@ public void test6() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -2568,7 +2568,7 @@ public void test7() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -3006,7 +3006,7 @@ public void test8() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -3445,7 +3445,7 @@ public void test9() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -3898,7 +3898,7 @@ public void test10() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -4410,7 +4410,7 @@ public void test11() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -4925,7 +4925,7 @@ public void test12() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -5440,7 +5440,7 @@ public void test13() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -6015,7 +6015,7 @@ public void test14() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -6527,7 +6527,7 @@ public void test15() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -6788,7 +6788,7 @@ public void test16() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -7119,7 +7119,7 @@ public void test17() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -7601,7 +7601,7 @@ public void test18() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -8040,7 +8040,7 @@ public void test19() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -8345,7 +8345,7 @@ public void test20() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -8512,7 +8512,7 @@ public void test21() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -8738,7 +8738,7 @@ public void test22() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -9009,7 +9009,7 @@ public void test23() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithEmptyPageTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithEmptyPageTest.java index 7e9c25316a548..f4d117458a095 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithEmptyPageTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithEmptyPageTest.java @@ -122,7 +122,7 @@ public void test1() throws IOException, IllegalPathException { } catch (Exception e) { Assert.fail(); } - TsFileResource result = tsFileManager.getTsFileList(true).get(0); + TsFileResource result = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); result.buildDeviceTimeIndex(); Assert.assertEquals(20, ((long) result.getStartTime(device).get())); Assert.assertEquals(50, ((long) result.getEndTime(device).get())); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java index f62480c55291f..2fccf21f20a13 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java @@ -124,7 +124,7 @@ public void test1() throws MetadataException, IOException, WriteProcessException 0); Assert.assertTrue(task.start()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); TsFileSequenceReader reader = new TsFileSequenceReader(targetFile.getTsFilePath()); validateSingleTsFileWithNonAlignedSeries(reader); } @@ -202,7 +202,7 @@ public void test2() throws MetadataException, IOException, WriteProcessException 0); Assert.assertTrue(task.start()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); TsFileSequenceReader reader = new TsFileSequenceReader(targetFile.getTsFilePath()); validateSingleTsFileWithNonAlignedSeries(reader); } @@ -276,7 +276,7 @@ public void test3() throws MetadataException, IOException, WriteProcessException 0); Assert.assertTrue(task.start()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); TsFileSequenceReader reader = new TsFileSequenceReader(targetFile.getTsFilePath()); // validateSingleTsFileWithNonAlignedSeries(reader); } @@ -327,7 +327,7 @@ public void test4() throws MetadataException, IOException, WriteProcessException 0); Assert.assertTrue(task.start()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); TsFileSequenceReader reader = new TsFileSequenceReader(targetFile.getTsFilePath()); // validateSingleTsFileWithNonAlignedSeries(reader); } @@ -405,7 +405,7 @@ public void test5() throws MetadataException, IOException, WriteProcessException 0); Assert.assertTrue(task.start()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); TsFileSequenceReader reader = new TsFileSequenceReader(targetFile.getTsFilePath()); // validateSingleTsFileWithNonAlignedSeries(reader); } @@ -479,7 +479,7 @@ public void test6() throws MetadataException, IOException, WriteProcessException 0); Assert.assertTrue(task.start()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); TsFileSequenceReader reader = new TsFileSequenceReader(targetFile.getTsFilePath()); validateSingleTsFileWithAlignedSeries(reader); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCrossCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCrossCompactionPerformerTest.java index 25f3acd89ee62..4928d000b2a59 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCrossCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCrossCompactionPerformerTest.java @@ -4764,8 +4764,8 @@ public void testCompactionWithMinTimestamp() throws IOException { 1000, 0); Assert.assertTrue(task.start()); - TsFileResource target1 = tsFileManager.getTsFileList(true).get(0); - TsFileResource target2 = tsFileManager.getTsFileList(true).get(1); + TsFileResource target1 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); + TsFileResource target2 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); Assert.assertEquals(1, target1.getDevices().size()); Assert.assertEquals(1, target2.getDevices().size()); Assert.assertEquals( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java index d75d7c4ede795..3c056e5eb303b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java @@ -156,8 +156,8 @@ public void testSeqInnerSpaceCompactionWithSameTimeseries() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); count = 0; @@ -240,7 +240,7 @@ public void testSeqInnerSpaceCompactionWithDifferentTimeseries() throws Exceptio Assert.assertTrue(task.start()); Assert.assertEquals(0, FileReaderManager.getInstance().getClosedFileReaderMap().size()); Assert.assertEquals(0, FileReaderManager.getInstance().getUnclosedFileReaderMap().size()); - List targetResources = tsFileManager.getTsFileList(true); + List targetResources = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); validateSeqFiles(true); assertEquals( @@ -308,8 +308,8 @@ public void testSeqInnerSpaceCompactionWithDifferentTimeseries() throws Exceptio path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -411,7 +411,7 @@ public void testSeqInnerSpaceCompactionWithFileTimeIndex() throws Exception { Assert.assertTrue(task.start()); Assert.assertEquals(0, FileReaderManager.getInstance().getClosedFileReaderMap().size()); Assert.assertEquals(0, FileReaderManager.getInstance().getUnclosedFileReaderMap().size()); - List targetResources = tsFileManager.getTsFileList(true); + List targetResources = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); validateSeqFiles(true); assertEquals( @@ -479,8 +479,8 @@ public void testSeqInnerSpaceCompactionWithFileTimeIndex() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -555,7 +555,7 @@ public void testUnSeqInnerSpaceCompactionWithSameTimeseries() throws Exception { Assert.assertTrue(task.start()); Assert.assertEquals(0, FileReaderManager.getInstance().getClosedFileReaderMap().size()); Assert.assertEquals(0, FileReaderManager.getInstance().getUnclosedFileReaderMap().size()); - List targetResources = tsFileManager.getTsFileList(false); + List targetResources = tsFileManager.getTsFileList(false, COMPACTION_TEST_SG); validateSeqFiles(false); for (int i = 0; i < 2; i++) { @@ -570,8 +570,8 @@ public void testUnSeqInnerSpaceCompactionWithSameTimeseries() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -679,8 +679,8 @@ public void testUnSeqInnerSpaceCompactionWithDifferentTimeseries() throws Except path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -818,8 +818,8 @@ public void testUnSeqInnerSpaceCompactionWithAllDataDeletedInTimeseries() throws path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -947,8 +947,8 @@ public void testUnSeqInnerSpaceCompactionWithAllDataDeletedInDevice() throws Exc path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -1062,8 +1062,8 @@ public void testUnSeqInnerSpaceCompactionWithAllDataDeletedInTargetFile() throws path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -1134,7 +1134,7 @@ public void testAlignedSeqInnerSpaceCompactionWithSameTimeseries() throws Except Assert.assertTrue(task.start()); Assert.assertEquals(0, FileReaderManager.getInstance().getClosedFileReaderMap().size()); Assert.assertEquals(0, FileReaderManager.getInstance().getUnclosedFileReaderMap().size()); - List targetResources = tsFileManager.getTsFileList(true); + List targetResources = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); validateSeqFiles(true); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -1154,8 +1154,8 @@ public void testAlignedSeqInnerSpaceCompactionWithSameTimeseries() throws Except path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -1269,8 +1269,8 @@ public void testAlignedSeqInnerSpaceCompactionWithDifferentTimeseriesAndEmptyPag path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -1394,8 +1394,8 @@ public void testAlignedSeqInnerSpaceCompactionWithDifferentTimeseriesAndEmptyChu path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), false); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -1524,8 +1524,8 @@ public void testAlignedUnSeqInnerSpaceCompactionWithEmptyChunkAndEmptyPage() thr path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -1704,8 +1704,8 @@ public void testAlignedUnSeqInnerSpaceCompactionWithAllDataDeletedInTimeseries() path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -1864,8 +1864,8 @@ public void testAlignedUnSeqInnerSpaceCompactionWithAllDataDeletedInDevice() thr path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -1976,8 +1976,8 @@ public void testAlignedUnSeqInnerSpaceCompactionWithSameTimeseries() throws Exce path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), - tsFileManager.getTsFileList(false), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG), true); int count = 0; while (tsBlockReader.hasNextBatch()) { @@ -2059,7 +2059,7 @@ public void testMergeAlignedSeriesTimeValuePairFromDifferentFiles() 0, tsFileManager, unseqResources, false, new FastCompactionPerformer(false), 0); Assert.assertTrue(task.start()); - TsFileResource targetResource = tsFileManager.getTsFileList(false).get(0); + TsFileResource targetResource = tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource.getTsFilePath())) { List chunkMetadataList = reader.getAlignedChunkMetadata( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastNonAlignedCrossCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastNonAlignedCrossCompactionTest.java index 64d70d30cdb10..fe6de7b30f9b2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastNonAlignedCrossCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastNonAlignedCrossCompactionTest.java @@ -335,7 +335,7 @@ public void test1() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -642,7 +642,7 @@ public void test2() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -954,7 +954,7 @@ public void test3() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -1278,7 +1278,7 @@ public void test4() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -1717,7 +1717,7 @@ public void test5() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -2163,7 +2163,7 @@ public void test6() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -2622,7 +2622,7 @@ public void test7() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -3063,7 +3063,7 @@ public void test8() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -3505,7 +3505,7 @@ public void test9() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -3960,7 +3960,7 @@ public void test10() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -4475,7 +4475,7 @@ public void test11() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -4993,7 +4993,7 @@ public void test12() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -5511,7 +5511,7 @@ public void test13() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -6089,7 +6089,7 @@ public void test14() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -6604,7 +6604,7 @@ public void test15() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -6871,7 +6871,7 @@ public void test16() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -7208,7 +7208,7 @@ public void test17() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -7696,7 +7696,7 @@ public void test18() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -8138,7 +8138,7 @@ public void test19() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -8449,7 +8449,7 @@ public void test20() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -8622,7 +8622,7 @@ public void test21() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -8854,7 +8854,7 @@ public void test22() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); @@ -9124,7 +9124,7 @@ public void test23() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java index 972db81b20a23..7b738c1486ad6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java @@ -447,7 +447,7 @@ public void testReadChunkPerformerWithEmptyTargetFile1() throws IOException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -473,7 +473,7 @@ public void testReadChunkPerformerWithEmptyTargetFile2() throws IOException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -531,7 +531,7 @@ public void testReadChunkPerformerWithEmptyTargetFile3() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -580,7 +580,7 @@ public void testReadChunkPerformerWithEmptyTargetFile4() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -629,7 +629,7 @@ public void testReadChunkPerformerWithEmptyTargetFile5() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -687,7 +687,7 @@ public void testReadChunkPerformerWithEmptyTargetFile6() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -715,7 +715,7 @@ public void testReadChunkPerformerWithNonEmptyTargetFile() throws IOException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -885,8 +885,8 @@ public void testCascadedDeletionDuringCompaction() throws IOException, Interrupt }) .start(); Assert.assertTrue(task.start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - tsFileManager.getTsFileList(true).get(0).getExclusiveModFile(); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getExclusiveModFile(); Assert.assertEquals(1, FileMetrics.getInstance().getModFileNum()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointAlignedCrossCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointAlignedCrossCompactionTest.java index f96cdf85cda10..9451fd6971373 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointAlignedCrossCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointAlignedCrossCompactionTest.java @@ -304,7 +304,7 @@ public void test1() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -609,7 +609,7 @@ public void test2() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -913,7 +913,7 @@ public void test3() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -1234,7 +1234,7 @@ public void test4() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -1670,7 +1670,7 @@ public void test5() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -2114,7 +2114,7 @@ public void test6() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -2571,7 +2571,7 @@ public void test7() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -3009,7 +3009,7 @@ public void test8() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -3448,7 +3448,7 @@ public void test9() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -3901,7 +3901,7 @@ public void test10() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -4413,7 +4413,7 @@ public void test11() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -4928,7 +4928,7 @@ public void test12() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -5443,7 +5443,7 @@ public void test13() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -6018,7 +6018,7 @@ public void test14() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -6530,7 +6530,7 @@ public void test15() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -6791,7 +6791,7 @@ public void test16() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -7122,7 +7122,7 @@ public void test17() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -7604,7 +7604,7 @@ public void test18() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8043,7 +8043,7 @@ public void test19() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8348,7 +8348,7 @@ public void test20() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8515,7 +8515,7 @@ public void test21() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8741,7 +8741,7 @@ public void test22() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -9012,7 +9012,7 @@ public void test23() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointNonAlignedCrossCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointNonAlignedCrossCompactionTest.java index 289ff7756b8e9..295acfafd4d25 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointNonAlignedCrossCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointNonAlignedCrossCompactionTest.java @@ -330,7 +330,7 @@ public void test1() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -631,7 +631,7 @@ public void test2() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -931,7 +931,7 @@ public void test3() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -1249,7 +1249,7 @@ public void test4() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -1682,7 +1682,7 @@ public void test5() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -2122,7 +2122,7 @@ public void test6() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -2575,7 +2575,7 @@ public void test7() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -3010,7 +3010,7 @@ public void test8() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -3446,7 +3446,7 @@ public void test9() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -3895,7 +3895,7 @@ public void test10() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -4404,7 +4404,7 @@ public void test11() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -4916,7 +4916,7 @@ public void test12() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -5428,7 +5428,7 @@ public void test13() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -6000,7 +6000,7 @@ public void test14() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -6509,7 +6509,7 @@ public void test15() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -6770,7 +6770,7 @@ public void test16() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -7101,7 +7101,7 @@ public void test17() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -7583,7 +7583,7 @@ public void test18() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8019,7 +8019,7 @@ public void test19() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8324,7 +8324,7 @@ public void test20() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8491,7 +8491,7 @@ public void test21() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8717,7 +8717,7 @@ public void test22() throws IOException, IllegalPathException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); @@ -8981,7 +8981,7 @@ public void test23() throws MetadataException, IOException { new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java index e9ea0025250ed..d39a125716f34 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java @@ -131,7 +131,7 @@ public void testAlter() throws IOException, WriteProcessException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); TimeseriesMetadata timeseriesMetadata = getTimeseriesMetadata(target, tableDevice, "s1"); Assert.assertEquals(1L, timeseriesMetadata.getStatistics().getStartTime()); @@ -145,7 +145,7 @@ public void testCannotAlter() throws IOException, WriteProcessException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); TimeseriesMetadata timeseriesMetadata = getTimeseriesMetadata(target, tableDevice, "s1"); if (!reverse) { Assert.assertEquals(1L, timeseriesMetadata.getStatistics().getStartTime()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTest.java index 96ddb03396ceb..81d3b6af14eb0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTest.java @@ -94,12 +94,21 @@ public void testCompactNonAlignedSeries() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + TsFileResourceUtils.validateTsFileDataCorrectness( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); Assert.assertEquals( - 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + 1, + ((long) + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getStartTime(device) + .get())); Assert.assertEquals( - 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); - TsFileResource tsFileResource = tsFileManager.getTsFileList(true).get(0); + 2, + ((long) + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getEndTime(device).get())); + TsFileResource tsFileResource = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath()); TsFileReader readTsFile = new TsFileReader(reader)) { @@ -123,12 +132,21 @@ public void testCompactAlignedSeries() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + TsFileResourceUtils.validateTsFileDataCorrectness( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); Assert.assertEquals( - 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + 1, + ((long) + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getStartTime(device) + .get())); Assert.assertEquals( - 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); - TsFileResource tsFileResource = tsFileManager.getTsFileList(true).get(0); + 2, + ((long) + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getEndTime(device).get())); + TsFileResource tsFileResource = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath()); TsFileReader readTsFile = new TsFileReader(reader)) { @@ -223,7 +241,7 @@ public void testAlterDataTypeWithAlignedSeriesWithTimeDeletion() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(41L, (long) target.getStartTime(device).get()); Assert.assertEquals(200L, (long) target.getEndTime(device).get()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchAlterableDataTypeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchAlterableDataTypeTest.java index cac37c3f385f9..b658e1b8506d8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchAlterableDataTypeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchAlterableDataTypeTest.java @@ -89,11 +89,20 @@ public void testCompactNonAlignedSeries() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + TsFileResourceUtils.validateTsFileDataCorrectness( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); Assert.assertEquals( - 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + 1, + ((long) + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getStartTime(device) + .get())); Assert.assertEquals( - 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + 2, + ((long) + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getEndTime(device).get())); } @Test @@ -104,11 +113,20 @@ public void testCompactAlignedSeries() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + TsFileResourceUtils.validateTsFileDataCorrectness( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); Assert.assertEquals( - 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + 1, + ((long) + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getStartTime(device) + .get())); Assert.assertEquals( - 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + 2, + ((long) + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getEndTime(device).get())); } private void generateDataTypeNotMatchFilesWithNonAlignedSeries() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchTest.java index 3002ef8981aaf..678cc96f69ccc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchTest.java @@ -89,11 +89,20 @@ public void testCompactNonAlignedSeries() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + TsFileResourceUtils.validateTsFileDataCorrectness( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); Assert.assertEquals( - 2, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + 2, + ((long) + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getStartTime(device) + .get())); Assert.assertEquals( - 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + 2, + ((long) + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getEndTime(device).get())); } @Test @@ -112,11 +121,20 @@ public void testCompactNonAlignedSeries2() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + TsFileResourceUtils.validateTsFileDataCorrectness( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); Assert.assertEquals( - 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + 1, + ((long) + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getStartTime(device) + .get())); Assert.assertEquals( - 1, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + 1, + ((long) + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getEndTime(device).get())); } @Test @@ -127,11 +145,20 @@ public void testCompactAlignedSeries() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + TsFileResourceUtils.validateTsFileDataCorrectness( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); Assert.assertEquals( - 2, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + 2, + ((long) + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getStartTime(device) + .get())); Assert.assertEquals( - 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + 2, + ((long) + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getEndTime(device).get())); } @Test @@ -154,11 +181,20 @@ public void testCompactAlignedSeries2() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + TsFileResourceUtils.validateTsFileDataCorrectness( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); Assert.assertEquals( - 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + 1, + ((long) + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getStartTime(device) + .get())); Assert.assertEquals( - 1, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + 1, + ((long) + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).getEndTime(device).get())); } private void generateDataTypeNotMatchFilesWithNonAlignedSeries() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java index 644ddf77e1f24..1c0b9c09df496 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java @@ -2152,7 +2152,7 @@ public void testTsFileValidationWithFileTimeIndex() // set the end time of d1 in the first seq file to 1100 tsFileManager - .getTsFileList(true) + .getTsFileList(true, COMPACTION_TEST_SG) .get(0) .updateEndTime( IDeviceID.Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d1"), @@ -2160,14 +2160,14 @@ public void testTsFileValidationWithFileTimeIndex() // set the end time of d1 in the second seq file to 1200 tsFileManager - .getTsFileList(true) + .getTsFileList(true, COMPACTION_TEST_SG) .get(1) .updateStartTime( IDeviceID.Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d1"), 1200L); for (int i = 1; i < seqResources.size(); i++) { - tsFileManager.getTsFileList(true).get(i).degradeTimeIndex(); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(i).degradeTimeIndex(); } // meet overlap files @@ -2175,10 +2175,10 @@ public void testTsFileValidationWithFileTimeIndex() TsFileResourceUtils.validateTsFileResourcesHasNoOverlap( tsFileManager.getOrCreateSequenceListByTimePartition(0).getArrayList())); - tsFileManager.getTsFileList(true).get(0).deserialize(); - tsFileManager.getTsFileList(true).get(1).deserialize(); - tsFileManager.getTsFileList(true).get(0).degradeTimeIndex(); - tsFileManager.getTsFileList(true).get(1).degradeTimeIndex(); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).deserialize(); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1).deserialize(); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).degradeTimeIndex(); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1).degradeTimeIndex(); Assert.assertTrue( TsFileResourceUtils.validateTsFileResourcesHasNoOverlap( tsFileManager.getOrCreateSequenceListByTimePartition(0).getArrayList())); @@ -2282,11 +2282,14 @@ public void testCompactionSchedule() throws Exception { Assert.assertEquals(2, pairs.get(0).getSeqFiles().size()); Assert.assertEquals(1, pairs.get(0).getUnseqFiles().size()); Assert.assertEquals( - tsFileManager.getTsFileList(true).get(4), pairs.get(0).getSeqFiles().get(0)); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(4), + pairs.get(0).getSeqFiles().get(0)); Assert.assertEquals( - tsFileManager.getTsFileList(true).get(5), pairs.get(0).getSeqFiles().get(1)); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(5), + pairs.get(0).getSeqFiles().get(1)); Assert.assertEquals( - tsFileManager.getTsFileList(false).get(0), pairs.get(0).getUnseqFiles().get(0)); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + pairs.get(0).getUnseqFiles().get(0)); // target file of first compaction task can be selected to participate in another inner // compaction task diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java index 5478d05004dff..da231ce6786aa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java @@ -173,7 +173,8 @@ public void testInsertionCompactionWithCachedDeviceInfoAndUnclosedResource() CompactionScheduler.tryToSubmitInsertionCompactionTask(tsFileManager, 0, phaser, context); Assert.assertEquals(0, submitTaskNum); Assert.assertTrue( - TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(tsFileManager.getTsFileList(true))); + TsFileResourceUtils.validateTsFileResourcesHasNoOverlap( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))); } @Test @@ -1090,23 +1091,28 @@ public void testInsertionSelectorWithNoSeqFiles() throws MergeException, IOExcep InsertionCrossCompactionTaskResource taskResource; int i = 1; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { taskResource = selector.selectOneInsertionTask( new CrossSpaceCompactionCandidate( - tsFileManager.getTsFileList(true), tsFileManager.getTsFileList(false))); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG))); Assert.assertTrue(taskResource.isValid()); Assert.assertEquals( - tsFileManager.getTsFileList(false).get(0), taskResource.firstUnSeqFileInParitition); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + taskResource.firstUnSeqFileInParitition); Assert.assertEquals( - tsFileManager.getTsFileList(false).get(0), taskResource.toInsertUnSeqFile); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + taskResource.toInsertUnSeqFile); Assert.assertEquals(null, taskResource.nextSeqFile); if (i == 1) { Assert.assertEquals(null, taskResource.prevSeqFile); } else if (i == 2) { - Assert.assertEquals(tsFileManager.getTsFileList(true).get(0), taskResource.prevSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0), taskResource.prevSeqFile); } else { - Assert.assertEquals(tsFileManager.getTsFileList(true).get(1), taskResource.prevSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1), taskResource.prevSeqFile); } InsertionCrossSpaceCompactionTask task = new InsertionCrossSpaceCompactionTask( @@ -1118,8 +1124,8 @@ public void testInsertionSelectorWithNoSeqFiles() throws MergeException, IOExcep task.start(); i++; } - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(0, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } @Test @@ -1304,10 +1310,15 @@ public void testInsertionIntoCompactingSeqFiles() throws IOException, MergeExcep InsertionCrossCompactionTaskResource task = selector.selectOneInsertionTask( new CrossSpaceCompactionCandidate(seqResources, unseqResources)); - Assert.assertEquals(tsFileManager.getTsFileList(false).get(0), task.toInsertUnSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(true).get(1), task.prevSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(true).get(2), task.nextSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(false).get(0), task.firstUnSeqFileInParitition); + Assert.assertEquals( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), task.toInsertUnSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1), task.prevSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(2), task.nextSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + task.firstUnSeqFileInParitition); // seq file 1 ~ 3 is compaction candidate for (int i = 0; i < 3; i++) { @@ -1316,10 +1327,15 @@ public void testInsertionIntoCompactingSeqFiles() throws IOException, MergeExcep task = selector.selectOneInsertionTask( new CrossSpaceCompactionCandidate(seqResources, unseqResources)); - Assert.assertEquals(tsFileManager.getTsFileList(false).get(0), task.toInsertUnSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(true).get(3), task.prevSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(true).get(4), task.nextSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(false).get(0), task.firstUnSeqFileInParitition); + Assert.assertEquals( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), task.toInsertUnSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(3), task.prevSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(4), task.nextSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + task.firstUnSeqFileInParitition); // all seq file are compacting for (TsFileResource resource : seqResources) { @@ -1579,23 +1595,28 @@ public void testInsertionSelectorWithNoSeqFilesAndFileTimeIndex() InsertionCrossCompactionTaskResource taskResource; int i = 1; - while (tsFileManager.getTsFileList(false).size() > 0) { + while (tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size() > 0) { taskResource = selector.selectOneInsertionTask( new CrossSpaceCompactionCandidate( - tsFileManager.getTsFileList(true), tsFileManager.getTsFileList(false))); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG))); Assert.assertTrue(taskResource.isValid()); Assert.assertEquals( - tsFileManager.getTsFileList(false).get(0), taskResource.firstUnSeqFileInParitition); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + taskResource.firstUnSeqFileInParitition); Assert.assertEquals( - tsFileManager.getTsFileList(false).get(0), taskResource.toInsertUnSeqFile); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + taskResource.toInsertUnSeqFile); Assert.assertEquals(null, taskResource.nextSeqFile); if (i == 1) { Assert.assertEquals(null, taskResource.prevSeqFile); } else if (i == 2) { - Assert.assertEquals(tsFileManager.getTsFileList(true).get(0), taskResource.prevSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0), taskResource.prevSeqFile); } else { - Assert.assertEquals(tsFileManager.getTsFileList(true).get(1), taskResource.prevSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1), taskResource.prevSeqFile); } InsertionCrossSpaceCompactionTask task = new InsertionCrossSpaceCompactionTask( @@ -1607,8 +1628,8 @@ public void testInsertionSelectorWithNoSeqFilesAndFileTimeIndex() task.start(); i++; } - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(0, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } @Test @@ -1812,10 +1833,15 @@ public void testInsertionIntoCompactingSeqFilesAndFileTimeIndex() InsertionCrossCompactionTaskResource task = selector.selectOneInsertionTask( new CrossSpaceCompactionCandidate(seqResources, unseqResources)); - Assert.assertEquals(tsFileManager.getTsFileList(false).get(0), task.toInsertUnSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(true).get(1), task.prevSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(true).get(2), task.nextSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(false).get(0), task.firstUnSeqFileInParitition); + Assert.assertEquals( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), task.toInsertUnSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1), task.prevSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(2), task.nextSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + task.firstUnSeqFileInParitition); // seq file 1 ~ 3 is compaction candidate for (int i = 0; i < 3; i++) { @@ -1824,10 +1850,15 @@ public void testInsertionIntoCompactingSeqFilesAndFileTimeIndex() task = selector.selectOneInsertionTask( new CrossSpaceCompactionCandidate(seqResources, unseqResources)); - Assert.assertEquals(tsFileManager.getTsFileList(false).get(0), task.toInsertUnSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(true).get(3), task.prevSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(true).get(4), task.nextSeqFile); - Assert.assertEquals(tsFileManager.getTsFileList(false).get(0), task.firstUnSeqFileInParitition); + Assert.assertEquals( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), task.toInsertUnSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(3), task.prevSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(4), task.nextSeqFile); + Assert.assertEquals( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0), + task.firstUnSeqFileInParitition); // all seq file are compacting for (TsFileResource resource : seqResources) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java index 73c0bc2a721c3..edad285e059c9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java @@ -123,13 +123,13 @@ public void test1() throws IOException, InterruptedException { task.setSourceFilesToCompactionCandidate(); candidateCompactionTaskQueue.put(task); Assert.assertTrue(worker.processOneCompactionTask(candidateCompactionTaskQueue.take())); - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(seqResource1, tsFileManager.getTsFileList(true).get(0)); - Assert.assertEquals(seqResource2, tsFileManager.getTsFileList(true).get(2)); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(1); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(seqResource1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); + Assert.assertEquals(seqResource2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(2)); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime(); Assert.assertEquals(2, timestamp); - Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty()); + Assert.assertTrue(tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).isEmpty()); Assert.assertEquals( tsFileManager.size(true) + tsFileManager.size(false), TsFileResourceManager.getInstance().getPriorityQueueSize()); @@ -168,10 +168,10 @@ public void test2() throws IOException, InterruptedException { task.setSourceFilesToCompactionCandidate(); candidateCompactionTaskQueue.put(task); Assert.assertTrue(worker.processOneCompactionTask(candidateCompactionTaskQueue.take())); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(seqResource1, tsFileManager.getTsFileList(true).get(0)); - Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(1); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(seqResource1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); + Assert.assertTrue(tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).isEmpty()); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime(); Assert.assertEquals(2, timestamp); Assert.assertEquals( @@ -212,10 +212,10 @@ public void test3() throws IOException, InterruptedException { task.setSourceFilesToCompactionCandidate(); candidateCompactionTaskQueue.put(task); Assert.assertTrue(worker.processOneCompactionTask(candidateCompactionTaskQueue.take())); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(seqResource1, tsFileManager.getTsFileList(true).get(1)); - Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(seqResource1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1)); + Assert.assertTrue(tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).isEmpty()); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime(); Assert.assertEquals(0, timestamp); Assert.assertEquals( @@ -261,13 +261,13 @@ public void test4() throws IOException, InterruptedException { task.setSourceFilesToCompactionCandidate(); candidateCompactionTaskQueue.put(task); Assert.assertTrue(worker.processOneCompactionTask(candidateCompactionTaskQueue.take())); - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(seqResource1, tsFileManager.getTsFileList(true).get(0)); - Assert.assertEquals(seqResource2, tsFileManager.getTsFileList(true).get(2)); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(1); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(seqResource1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0)); + Assert.assertEquals(seqResource2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(2)); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime(); Assert.assertEquals(4, timestamp); - Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty()); + Assert.assertTrue(tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).isEmpty()); Assert.assertEquals( tsFileManager.size(true) + tsFileManager.size(false), TsFileResourceManager.getInstance().getPriorityQueueSize()); @@ -301,9 +301,9 @@ public void test5() throws IOException, InterruptedException { task.setSourceFilesToCompactionCandidate(); candidateCompactionTaskQueue.put(task); Assert.assertTrue(worker.processOneCompactionTask(candidateCompactionTaskQueue.take())); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty()); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertTrue(tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).isEmpty()); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime(); Assert.assertEquals(2, timestamp); Assert.assertEquals( @@ -349,13 +349,13 @@ public void testInsertionCompactionSchedule() throws IOException, InterruptedExc tsFileManager.addAll(seqResources, true); tsFileManager.addAll(unseqResources, false); Assert.assertEquals(2, dataRegion.executeInsertionCompaction()); - Assert.assertEquals(4, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(4, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); - TsFileResource targetFile1 = tsFileManager.getTsFileList(true).get(1); + TsFileResource targetFile1 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); long timestamp = TsFileNameGenerator.getTsFileName(targetFile1.getTsFile().getName()).getTime(); Assert.assertEquals(4, timestamp); - TsFileResource targetFile2 = tsFileManager.getTsFileList(true).get(2); + TsFileResource targetFile2 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(2); timestamp = TsFileNameGenerator.getTsFileName(targetFile2.getTsFile().getName()).getTime(); Assert.assertEquals(6, timestamp); Assert.assertEquals( @@ -388,15 +388,15 @@ public void testInsertionCompactionScheduleWithEmptySeqSpace() tsFileManager.addAll(seqResources, true); tsFileManager.addAll(unseqResources, false); Assert.assertEquals(3, dataRegion.executeInsertionCompaction()); - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); - TsFileResource targetFile1 = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile1 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); long timestamp = TsFileNameGenerator.getTsFileName(targetFile1.getTsFile().getName()).getTime(); Assert.assertEquals(2, timestamp); - TsFileResource targetFile2 = tsFileManager.getTsFileList(true).get(1); + TsFileResource targetFile2 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); timestamp = TsFileNameGenerator.getTsFileName(targetFile2.getTsFile().getName()).getTime(); Assert.assertEquals(3, timestamp); - TsFileResource targetFile3 = tsFileManager.getTsFileList(true).get(2); + TsFileResource targetFile3 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(2); timestamp = TsFileNameGenerator.getTsFileName(targetFile3.getTsFile().getName()).getTime(); Assert.assertEquals(4, timestamp); Assert.assertEquals( @@ -439,12 +439,12 @@ public void testInsertionCompactionScheduleWithEmptySeqSpace2() tsFileManager.addAll(seqResources, true); tsFileManager.addAll(unseqResources, false); Assert.assertEquals(2, dataRegion.executeInsertionCompaction()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); - TsFileResource targetFile1 = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile1 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); long timestamp = TsFileNameGenerator.getTsFileName(targetFile1.getTsFile().getName()).getTime(); Assert.assertEquals(1, timestamp); - TsFileResource targetFile2 = tsFileManager.getTsFileList(true).get(1); + TsFileResource targetFile2 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); timestamp = TsFileNameGenerator.getTsFileName(targetFile2.getTsFile().getName()).getTime(); Assert.assertEquals(2, timestamp); Assert.assertEquals( @@ -487,7 +487,7 @@ public void testInsertionCompactionScheduleWithMultiTimePartitions1() tsFileManager.getOrCreateUnsequenceListByTimePartition(2808).keepOrderInsert(unseqResource3); Assert.assertEquals(3, dataRegion.executeInsertionCompaction()); - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); Assert.assertEquals(2, tsFileManager.getOrCreateSequenceListByTimePartition(0).size()); Assert.assertEquals(1, tsFileManager.getOrCreateSequenceListByTimePartition(2808).size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java index 4e81e3ee87cf7..3e03e708d0aaf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java @@ -276,7 +276,7 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInTimeseries() thro path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -506,7 +506,7 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInOneTargetFile() t path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java index ab9dd90e778a9..57861fef8ba77 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java @@ -271,7 +271,7 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInTimeseries() thro path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -501,7 +501,7 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInOneTargetFile() t path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionMoreDataTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionMoreDataTest.java index 09e593766c86d..ef7b54182af1d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionMoreDataTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionMoreDataTest.java @@ -171,7 +171,7 @@ public void testSensorWithTwoOrThreeNode() new SeriesDataBlockReader( path, TEST_QUERY_FI_CONTEXT, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionSchedulerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionSchedulerTest.java index 340270514713f..993ebfb747957 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionSchedulerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionSchedulerTest.java @@ -103,7 +103,7 @@ public void testFileSelector1() } catch (Exception e) { } - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -134,7 +134,7 @@ public void testFileSelector2() e.printStackTrace(); } } - Assert.assertEquals(4, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(4, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -163,6 +163,6 @@ public void testFileSelectorWithUnclosedFile() e.printStackTrace(); } } - Assert.assertEquals(4, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(4, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java index 2ede5aa26f6cc..83a5394bb35bc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java @@ -126,7 +126,7 @@ public void testSelectAllFiles() throws IOException { Assert.assertTrue(task.start()); Assert.assertEquals(10, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(10, task.getAllSourceTsFiles().size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -152,12 +152,12 @@ public void testSelectWithFileNumLimit() throws IOException { Assert.assertTrue(task1.start()); Assert.assertEquals(8, task1.getSelectedTsFileResourceList().size()); Assert.assertEquals(8, task1.getAllSourceTsFiles().size()); - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); InnerSpaceCompactionTask task2 = innerSpaceCompactionTasks.get(1); Assert.assertEquals(2, task2.getSelectedTsFileResourceList().size()); Assert.assertEquals(2, task2.getAllSourceTsFiles().size()); Assert.assertTrue(task2.start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -224,7 +224,8 @@ public void testSkipSomeFilesAndRenamePreviousFiles() throws IOException { Assert.assertTrue(task.start()); Assert.assertEquals(2, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(10, task.getAllSourceTsFiles().size()); - List filesAfterCompaction = tsFileManager.getTsFileList(true); + List filesAfterCompaction = + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); Assert.assertEquals(9, filesAfterCompaction.size()); Assert.assertEquals(0, filesAfterCompaction.get(0).getTsFileID().fileVersion); Assert.assertEquals(101L, filesAfterCompaction.get(0).getFileStartTime()); @@ -266,7 +267,8 @@ public void testSkipSomeFilesAndRenamePreviousFiles2() throws IOException { // select resource1, resource3, resource5 Assert.assertEquals(3, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(5, task.getAllSourceTsFiles().size()); - List filesAfterCompaction = tsFileManager.getTsFileList(true); + List filesAfterCompaction = + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); Assert.assertEquals(5, filesAfterCompaction.size()); Assert.assertEquals( 5, filesAfterCompaction.get(filesAfterCompaction.size() - 1).getTsFileID().fileVersion); @@ -319,7 +321,8 @@ public void testSkipSomeFiles() throws IOException { Assert.assertTrue(task.start()); Assert.assertEquals(3, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(5, task.getAllSourceTsFiles().size()); - List filesAfterCompaction = tsFileManager.getTsFileList(true); + List filesAfterCompaction = + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); Assert.assertEquals(3, filesAfterCompaction.size()); Assert.assertEquals( 5, filesAfterCompaction.get(filesAfterCompaction.size() - 1).getTsFileID().fileVersion); @@ -370,7 +373,8 @@ public void testSkipSomeFilesAndRenamePreviousFilesWithCompactionMods() Assert.assertTrue(task.start()); Assert.assertEquals(2, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(10, task.getAllSourceTsFiles().size()); - List filesAfterCompaction = tsFileManager.getTsFileList(true); + List filesAfterCompaction = + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); Assert.assertEquals(9, filesAfterCompaction.size()); Assert.assertEquals(0, filesAfterCompaction.get(0).getTsFileID().fileVersion); Assert.assertEquals(101L, filesAfterCompaction.get(0).getFileStartTime()); @@ -417,7 +421,8 @@ public void testAllTargetFilesEmpty() throws IOException, IllegalPathException { Assert.assertTrue(task.start()); Assert.assertEquals(2, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(2, task.getAllSourceTsFiles().size()); - List filesAfterCompaction = tsFileManager.getTsFileList(true); + List filesAfterCompaction = + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); Assert.assertEquals(0, filesAfterCompaction.size()); } @@ -471,7 +476,8 @@ public void testAllTargetFilesEmptyWithSkippedSourceFiles() Assert.assertTrue(task.start()); Assert.assertEquals(3, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(5, task.getAllSourceTsFiles().size()); - List filesAfterCompaction = tsFileManager.getTsFileList(true); + List filesAfterCompaction = + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); Assert.assertEquals(2, filesAfterCompaction.size()); Assert.assertEquals( 4, filesAfterCompaction.get(filesAfterCompaction.size() - 1).getTsFileID().fileVersion); @@ -503,7 +509,8 @@ public void testSelectFilesInOtherLevel() throws IOException { Assert.assertTrue(task.start()); Assert.assertEquals(6, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(6, task.getAllSourceTsFiles().size()); - List filesAfterCompaction = tsFileManager.getTsFileList(true); + List filesAfterCompaction = + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); Assert.assertEquals(5, filesAfterCompaction.size()); Assert.assertEquals(0, filesAfterCompaction.get(0).getTsFileID().fileVersion); Assert.assertEquals(1L, filesAfterCompaction.get(0).getFileStartTime()); @@ -540,7 +547,8 @@ public void testSkipToPreviousIndexAndSelectSkippedFiles() throws IOException { Assert.assertTrue(task.start()); Assert.assertEquals(5, task.getSelectedTsFileResourceList().size()); Assert.assertEquals(5, task.getAllSourceTsFiles().size()); - List filesAfterCompaction = tsFileManager.getTsFileList(true); + List filesAfterCompaction = + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); Assert.assertEquals(6, filesAfterCompaction.size()); Assert.assertEquals(5, filesAfterCompaction.get(5).getTsFileID().fileVersion); Assert.assertEquals(501L, filesAfterCompaction.get(5).getFileStartTime()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java index cc9e713c94c6d..1652d48bd0625 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java @@ -109,7 +109,7 @@ public void testCompactionRecoverWithUncompletedTargetFileAndLog() throws Except path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -197,7 +197,7 @@ public void testCompactionRecoverWithUncompletedTargetFileAndLog() throws Except path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); count = 0; @@ -229,7 +229,7 @@ public void testRecoverWithAllSourceFilesExisted() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -305,7 +305,7 @@ public void testRecoverWithAllSourceFilesExisted() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); count = 0; @@ -337,7 +337,7 @@ public void testRecoverWithAllSourceFilesExistedAndTargetFileNotExist() throws E path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -415,7 +415,7 @@ public void testRecoverWithAllSourceFilesExistedAndTargetFileNotExist() throws E path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); count = 0; @@ -445,7 +445,7 @@ public void testRecoverWithoutAllSourceFilesExisted() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -518,7 +518,7 @@ public void testRecoverWithoutAllSourceFilesExisted() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true).subList(3, 6), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).subList(3, 6), new ArrayList<>(), true); count = 0; @@ -772,7 +772,7 @@ public void testRecoverCompleteTargetFileAndCompactionLog() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -838,7 +838,7 @@ public void testRecoverCompleteTargetFileAndCompactionLog() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true).subList(0, 5), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).subList(0, 5), new ArrayList<>(), true); count = 0; @@ -870,7 +870,7 @@ public void testCompactionRecoverWithCompletedTargetFileAndLog() throws Exceptio path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -929,7 +929,7 @@ public void testCompactionRecoverWithCompletedTargetFileAndLog() throws Exceptio path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); count = 0; @@ -962,7 +962,7 @@ public void testCompactionRecoverWithCompletedTargetFile() throws Exception { path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -1017,13 +1017,13 @@ public void testCompactionRecoverWithCompletedTargetFile() throws Exception { deviceIds[0] + TsFileConstant.PATH_SEPARATOR + measurementSchemas[0].getMeasurementName()); - logger.warn("TsFiles in list is {}", tsFileManager.getTsFileList(true)); + logger.warn("TsFiles in list is {}", tsFileManager.getTsFileList(true, COMPACTION_TEST_SG)); tsFilesReader = new SeriesDataBlockReader( path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); count = 0; @@ -1066,7 +1066,7 @@ public void testCompactionMergeRecoverMergeStartSourceLog() path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -1107,7 +1107,7 @@ public void testCompactionMergeRecoverMergeStartSequenceLog() path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; @@ -1162,7 +1162,7 @@ public void testCompactionMergeRecoverMergeStart() throws IOException, MetadataE path, FragmentInstanceContext.createFragmentInstanceContextForCompaction( EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()), - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), new ArrayList<>(), true); int count = 0; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java index 7774bbb609b3c..f222d2261723d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java @@ -124,7 +124,8 @@ public void testMoveOldVersionResourceFile() throws IOException { Assert.assertTrue(task.start()); TsFileResource resource2 = - new TsFileResource(tsFileManager.getTsFileList(false).get(0).getTsFile()); + new TsFileResource( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0).getTsFile()); resource2.deserialize(); Assert.assertEquals( ArrayDeviceTimeIndex.ARRAY_DEVICE_TIME_INDEX_TYPE, resource2.getTimeIndexType()); @@ -148,11 +149,11 @@ public void testRepairUnsortedDataBetweenPageWithNonAlignedSeries() throws IOExc RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0); task.start(); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); Assert.assertTrue( TsFileResourceUtils.validateTsFileDataCorrectness( - tsFileManager.getTsFileList(false).get(0))); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0))); } @Test @@ -173,11 +174,11 @@ public void testRepairUnsortedDataBetweenPageWithAlignedSeries() throws IOExcept RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0); task.start(); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); Assert.assertTrue( TsFileResourceUtils.validateTsFileDataCorrectness( - tsFileManager.getTsFileList(false).get(0))); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0))); } @Test @@ -202,11 +203,11 @@ public void testRepairUnsortedDataInOnePageWithNonAlignedSeries() throws IOExcep RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0); task.start(); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); Assert.assertTrue( TsFileResourceUtils.validateTsFileDataCorrectness( - tsFileManager.getTsFileList(false).get(0))); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0))); } @Test @@ -235,11 +236,11 @@ public void testRepairUnsortedDataInOnePageWithMultiNonAlignedSeries() throws IO RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0); task.start(); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); Assert.assertTrue( TsFileResourceUtils.validateTsFileDataCorrectness( - tsFileManager.getTsFileList(false).get(0))); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0))); } @Test @@ -264,11 +265,11 @@ public void testRepairUnsortedDataInOnePageWithUnseqFile() throws IOException { RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0); task.start(); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); Assert.assertTrue( TsFileResourceUtils.validateTsFileDataCorrectness( - tsFileManager.getTsFileList(false).get(0))); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0))); } @Test @@ -293,11 +294,11 @@ public void testRepairUnsortedDataInOnePageWithAlignedSeries() throws IOExceptio RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0); task.start(); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); Assert.assertTrue( TsFileResourceUtils.validateTsFileDataCorrectness( - tsFileManager.getTsFileList(false).get(0))); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0))); Assert.assertTrue( TsFileResourceUtils.validateTsFileResourcesHasNoOverlap( tsFileManager.getOrCreateSequenceListByTimePartition(0))); @@ -344,7 +345,7 @@ public void testMarkFileAndRepairWithInnerSeqSpaceCompactionTask() 0); Assert.assertFalse(task.start()); - for (TsFileResource resource : tsFileManager.getTsFileList(true)) { + for (TsFileResource resource : tsFileManager.getTsFileList(true, COMPACTION_TEST_SG)) { Assert.assertTrue(resource.getTsFileRepairStatus().isRepairCompactionCandidate()); } @@ -368,9 +369,9 @@ public void testMarkFileAndRepairWithInnerSeqSpaceCompactionTask() throw new RuntimeException(e); } } - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(2, tsFileManager.getTsFileList(false).size()); - for (TsFileResource resource : tsFileManager.getTsFileList(false)) { + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + for (TsFileResource resource : tsFileManager.getTsFileList(false, COMPACTION_TEST_SG)) { TsFileResourceUtils.validateTsFileDataCorrectness(resource); } Assert.assertTrue( @@ -419,7 +420,7 @@ public void testMarkFileAndRepairWithInnerUnSeqSpaceCompactionTask() 0); Assert.assertFalse(task.start()); - for (TsFileResource resource : tsFileManager.getTsFileList(true)) { + for (TsFileResource resource : tsFileManager.getTsFileList(true, COMPACTION_TEST_SG)) { Assert.assertTrue(resource.getTsFileRepairStatus().isRepairCompactionCandidate()); } @@ -443,9 +444,9 @@ public void testMarkFileAndRepairWithInnerUnSeqSpaceCompactionTask() throw new RuntimeException(e); } } - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(2, tsFileManager.getTsFileList(false).size()); - for (TsFileResource resource : tsFileManager.getTsFileList(false)) { + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + for (TsFileResource resource : tsFileManager.getTsFileList(false, COMPACTION_TEST_SG)) { TsFileResourceUtils.validateTsFileDataCorrectness(resource); } Assert.assertTrue( @@ -495,7 +496,7 @@ public void testMarkFileAndRepairWithCrossSpaceCompactionTask() 0); Assert.assertFalse(task.start()); - for (TsFileResource resource : tsFileManager.getTsFileList(true)) { + for (TsFileResource resource : tsFileManager.getTsFileList(true, COMPACTION_TEST_SG)) { Assert.assertTrue(resource.getTsFileRepairStatus().isRepairCompactionCandidate()); } @@ -520,9 +521,9 @@ public void testMarkFileAndRepairWithCrossSpaceCompactionTask() throw new RuntimeException(e); } } - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(2, tsFileManager.getTsFileList(false).size()); - for (TsFileResource resource : tsFileManager.getTsFileList(false)) { + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + for (TsFileResource resource : tsFileManager.getTsFileList(false, COMPACTION_TEST_SG)) { TsFileResourceUtils.validateTsFileDataCorrectness(resource); } } @@ -563,14 +564,14 @@ public void testRepairOverlapBetweenFile() throws IOException { RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, seqResource2, true, 0); Assert.assertTrue(task.start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); Assert.assertTrue( TsFileResourceUtils.validateTsFileDataCorrectness( - tsFileManager.getTsFileList(false).get(0))); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0))); Assert.assertTrue( TsFileResourceUtils.validateTsFileResourceCorrectness( - tsFileManager.getTsFileList(false).get(0))); + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0))); } @Test @@ -614,9 +615,9 @@ public void testRepairOverlapBetweenFileWithModFile() throws IOException, Illega RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, seqResource2, true, 0); Assert.assertTrue(task.start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); - TsFileResource targetResource = tsFileManager.getTsFileList(false).get(0); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + TsFileResource targetResource = tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0); Assert.assertTrue(TsFileResourceUtils.validateTsFileDataCorrectness(targetResource)); Assert.assertTrue(TsFileResourceUtils.validateTsFileResourceCorrectness(targetResource)); Assert.assertTrue(targetResource.anyModFileExists()); @@ -706,7 +707,7 @@ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException { RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(false).get(0); + TsFileResource target = tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { List chunkMetadataList = reader.getAlignedChunkMetadata( @@ -820,7 +821,7 @@ public void testSplitChunk() throws IOException { RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(false).get(0); + TsFileResource target = tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { List chunkMetadataList = reader.getAlignedChunkMetadata( @@ -864,7 +865,7 @@ public void testResourceFileLostDevices() throws IOException { RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(false).get(0); + TsFileResource target = tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { List devicesInTargetFile = reader.getAllDevices(); Assert.assertEquals(Arrays.asList(d1, d2), devicesInTargetFile); @@ -909,7 +910,8 @@ public void testQueryRepairResult() throws IOException, IllegalPathException { task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource2, true, 0); Assert.assertTrue(task.start()); - List sourceTsFileResources = tsFileManager.getTsFileList(false); + List sourceTsFileResources = + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG); List fullPaths = getPaths(sourceTsFileResources); Map> dataByQuery1 = CompactionCheckerUtils.getDataByQuery( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java index 3108a20468694..404228e9f2f19 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java @@ -127,8 +127,8 @@ public void testScheduleRepairInternalUnsortedFile() throws IOException { new UnsortedFileRepairTaskScheduler( Collections.singletonList(mockDataRegion), false, tempDir); scheduler.run(); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } @Test @@ -196,7 +196,7 @@ public void testRecoverRepairScheduleSkipRepairedTimePartitionAndMarkFile() thro new UnsortedFileRepairTaskScheduler( Collections.singletonList(mockDataRegion), true, tempDir); scheduler.run(); - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); // check whether the repair status is marked correctly Assert.assertTrue(seqResource3.getTsFileRepairStatus().isRepairCompactionCandidate()); } @@ -256,8 +256,8 @@ public void testScheduleRepairOverlapFileAndInternalUnsortedFile() throws IOExce new UnsortedFileRepairTaskScheduler( Collections.singletonList(mockDataRegion), false, tempDir); scheduler.run(); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(2, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } @Test @@ -303,8 +303,8 @@ public void testScheduleRepairOverlapFile() throws IOException { new UnsortedFileRepairTaskScheduler( Collections.singletonList(mockDataRegion), false, tempDir); scheduler.run(); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } @Test @@ -379,6 +379,6 @@ public void testRecoverRepairScheduleSkipRepairedTimePartitionWithDeletedFile() new UnsortedFileRepairTaskScheduler( Collections.singletonList(mockDataRegion), true, tempDir); scheduler.run(); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionRecoverTest.java index d587aa4ca344f..f40c2ed9f6eb6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionRecoverTest.java @@ -123,8 +123,8 @@ public void handExceptionWhenSettlingAllDeletedFilesWithOnlyAllDeletedFiles() Assert.assertFalse(resource.getCompactionModFile().exists()); } - Assert.assertEquals(3, tsFileManager.getTsFileList(false).size()); - Assert.assertEquals(6, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + Assert.assertEquals(6, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -178,8 +178,8 @@ public void handExceptionWhenSettlingAllDeletedFiles() Assert.assertFalse(resource.getCompactionModFile().exists()); } - Assert.assertEquals(3, tsFileManager.getTsFileList(false).size()); - Assert.assertEquals(6, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + Assert.assertEquals(6, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -254,12 +254,12 @@ public void handExceptionWhenSettlingPartialDeletedFilesWithAllSourceFileExisted Assert.assertFalse(resource.getCompactionModFile().exists()); } - Assert.assertEquals(3, tsFileManager.getTsFileList(false).size()); - Assert.assertEquals(6, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + Assert.assertEquals(6, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); Assert.assertTrue(!tsFileManager.contains(targetResource, false)); Assert.assertTrue(!tsFileManager.contains(targetResource, true)); // resource file exist - for (TsFileResource resource : tsFileManager.getTsFileList(false)) { + for (TsFileResource resource : tsFileManager.getTsFileList(false, COMPACTION_TEST_SG)) { Assert.assertTrue(resource.tsFileExists()); Assert.assertTrue(resource.anyModFileExists()); Assert.assertTrue(resource.resourceFileExists()); @@ -357,8 +357,8 @@ public void handExceptionWhenSettlingPartialDeletedFilesWithSomeSourceFileLosted Assert.assertTrue(targetResource.tsFileExists()); Assert.assertTrue(targetResource.anyModFileExists()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); - for (TsFileResource resource : tsFileManager.getTsFileList(false)) { + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + for (TsFileResource resource : tsFileManager.getTsFileList(false, COMPACTION_TEST_SG)) { Assert.assertTrue(resource.tsFileExists()); Assert.assertTrue(resource.anyModFileExists()); Assert.assertTrue(resource.resourceFileExists()); @@ -455,7 +455,7 @@ public void handExceptionWhenSettlingPartialDeletedFilesWithSomeSourceFileLosted Assert.assertFalse(targetResource.anyModFileExists()); Assert.assertFalse(targetResource.getCompactionModFile().exists()); - Assert.assertEquals(0, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionSelectorTest.java index b37997706de27..e06c91fd73a88 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionSelectorTest.java @@ -94,11 +94,13 @@ public void testSelectContinuousFileWithLightSelect() Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(unseqTasks.get(0).start()); - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(3, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); - Assert.assertFalse(tsFileManager.getTsFileList(true).get(0).anyModFileExists()); - Assert.assertFalse(tsFileManager.getTsFileList(false).get(0).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0).anyModFileExists()); // select second time seqTasks = settleSelector.selectSettleTask(seqResources); @@ -110,11 +112,13 @@ public void testSelectContinuousFileWithLightSelect() Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(unseqTasks.get(0).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(2, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); for (int i = 0; i < 2; i++) { - Assert.assertFalse(tsFileManager.getTsFileList(true).get(i).anyModFileExists()); - Assert.assertFalse(tsFileManager.getTsFileList(false).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(i).anyModFileExists()); } seqTasks = settleSelector.selectSettleTask(seqResources); @@ -164,8 +168,8 @@ public void testSelectUnContinuousFileWithLightSelect() Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(unseqTasks.get(0).start()); - Assert.assertEquals(--num, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(num, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(--num, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(num, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } List seqTasks = settleSelector.selectSettleTask(seqResources); @@ -232,7 +236,7 @@ public void testSelectContinuousFilesBaseOnDirtyRateByModsWithHeavySelect() Assert.assertEquals(0, seqTasks.get(0).getPartiallyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); - Assert.assertEquals(4, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(4, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); // select second time settleSelector = @@ -243,7 +247,7 @@ public void testSelectContinuousFilesBaseOnDirtyRateByModsWithHeavySelect() Assert.assertEquals(0, seqTasks.get(0).getFullyDirtyFiles().size()); Assert.assertEquals(3, seqTasks.get(0).getPartiallyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); // select third time seqTasks = settleSelector.selectSettleTask(seqResources); @@ -251,10 +255,11 @@ public void testSelectContinuousFilesBaseOnDirtyRateByModsWithHeavySelect() Assert.assertEquals(0, seqTasks.get(0).getFullyDirtyFiles().size()); Assert.assertEquals(1, seqTasks.get(0).getPartiallyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); for (int i = 0; i < 2; i++) { - Assert.assertFalse(tsFileManager.getTsFileList(true).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(i).anyModFileExists()); } } @@ -301,8 +306,9 @@ public void testSelectContinuousFileBaseOnDirtyDataRateWithHeavySelect() Assert.assertEquals(0, unseqTasks.get(0).getFullyDirtyFiles().size()); Assert.assertTrue(unseqTasks.get(0).start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); - Assert.assertFalse(tsFileManager.getTsFileList(false).get(0).anyModFileExists()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + Assert.assertFalse( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0).anyModFileExists()); // select third time // all seq files is partial_deleted @@ -321,7 +327,7 @@ public void testSelectContinuousFileBaseOnDirtyDataRateWithHeavySelect() Assert.assertEquals(0, seqTasks.get(0).getFullyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } // base on outdated too long @@ -369,12 +375,14 @@ public void testSelectContinuousFileBaseOnDirtyDataOutdatedTooLongWithHeavySelec Assert.assertTrue(seqTasks.get(1).start()); Assert.assertTrue(unseqTasks.get(0).start()); Assert.assertTrue(unseqTasks.get(1).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(2, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); for (int i = 0; i < 2; i++) { - Assert.assertFalse(tsFileManager.getTsFileList(true).get(i).anyModFileExists()); - Assert.assertFalse(tsFileManager.getTsFileList(false).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(i).anyModFileExists()); } // select third time @@ -461,7 +469,7 @@ public void testSelectUncontinuousFileBaseOnDirtyDataRateWithHeavySelect() Assert.assertEquals(2, seqTasks.get(1).getPartiallyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(seqTasks.get(1).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } /** @@ -528,7 +536,7 @@ public void testSelectFileBaseOnDirtyDataRateWithHeavySelect() Assert.assertTrue(seqTasks.get(1).start()); Assert.assertTrue(seqTasks.get(2).start()); Assert.assertTrue(seqTasks.get(3).start()); - Assert.assertEquals(8, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(8, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } /** @@ -594,7 +602,7 @@ public void testSelectFileBaseOnDirtyDataRateWithHeavySelect2() Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(seqTasks.get(1).start()); Assert.assertTrue(seqTasks.get(2).start()); - Assert.assertEquals(6, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(6, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } // endregion @@ -630,11 +638,13 @@ public void testSelectContinuousFileWithLightSelectAligned() Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(unseqTasks.get(0).start()); - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(3, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(3, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); - Assert.assertFalse(tsFileManager.getTsFileList(true).get(0).anyModFileExists()); - Assert.assertFalse(tsFileManager.getTsFileList(false).get(0).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0).anyModFileExists()); // select second time seqTasks = settleSelector.selectSettleTask(seqResources); @@ -646,11 +656,13 @@ public void testSelectContinuousFileWithLightSelectAligned() Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(unseqTasks.get(0).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(2, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); for (int i = 0; i < 2; i++) { - Assert.assertFalse(tsFileManager.getTsFileList(true).get(i).anyModFileExists()); - Assert.assertFalse(tsFileManager.getTsFileList(false).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(i).anyModFileExists()); } seqTasks = settleSelector.selectSettleTask(seqResources); @@ -701,8 +713,8 @@ public void testSelectUnContinuousFileWithLightSelectAligned() Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(unseqTasks.get(0).start()); - Assert.assertEquals(--num, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(num, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(--num, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(num, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); } List seqTasks = settleSelector.selectSettleTask(seqResources); @@ -769,7 +781,7 @@ public void testSelectContinuousFilesBaseOnDirtyRateByModsWithHeavySelectAligned Assert.assertEquals(0, seqTasks.get(0).getPartiallyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); - Assert.assertEquals(4, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(4, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); // select second time settleSelector = @@ -780,7 +792,7 @@ public void testSelectContinuousFilesBaseOnDirtyRateByModsWithHeavySelectAligned Assert.assertEquals(0, seqTasks.get(0).getFullyDirtyFiles().size()); Assert.assertEquals(3, seqTasks.get(0).getPartiallyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); // select third time seqTasks = settleSelector.selectSettleTask(seqResources); @@ -788,10 +800,11 @@ public void testSelectContinuousFilesBaseOnDirtyRateByModsWithHeavySelectAligned Assert.assertEquals(0, seqTasks.get(0).getFullyDirtyFiles().size()); Assert.assertEquals(1, seqTasks.get(0).getPartiallyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); for (int i = 0; i < 2; i++) { - Assert.assertFalse(tsFileManager.getTsFileList(true).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(i).anyModFileExists()); } } @@ -838,8 +851,9 @@ public void testSelectContinuousFileBaseOnDirtyDataRateWithHeavySelectAligned() Assert.assertEquals(0, unseqTasks.get(0).getFullyDirtyFiles().size()); Assert.assertTrue(unseqTasks.get(0).start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); - Assert.assertFalse(tsFileManager.getTsFileList(false).get(0).anyModFileExists()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); + Assert.assertFalse( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(0).anyModFileExists()); // select third time // all seq files is partial_deleted @@ -858,7 +872,7 @@ public void testSelectContinuousFileBaseOnDirtyDataRateWithHeavySelectAligned() Assert.assertEquals(0, seqTasks.get(0).getFullyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } // base on outdated too long @@ -906,12 +920,14 @@ public void testSelectContinuousFileBaseOnDirtyDataOutdatedTooLongWithHeavySelec Assert.assertTrue(seqTasks.get(1).start()); Assert.assertTrue(unseqTasks.get(0).start()); Assert.assertTrue(unseqTasks.get(1).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(2, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); for (int i = 0; i < 2; i++) { - Assert.assertFalse(tsFileManager.getTsFileList(true).get(i).anyModFileExists()); - Assert.assertFalse(tsFileManager.getTsFileList(false).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(i).anyModFileExists()); + Assert.assertFalse( + tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).get(i).anyModFileExists()); } // select third time @@ -998,7 +1014,7 @@ public void testSelectUncontinuousFileBaseOnDirtyDataRateWithHeavySelectAligned( Assert.assertEquals(2, seqTasks.get(1).getPartiallyDirtyFiles().size()); Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(seqTasks.get(1).start()); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(2, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } /** @@ -1066,7 +1082,7 @@ public void testSelectFileBaseOnDirtyDataRateWithHeavySelectAligned() Assert.assertTrue(seqTasks.get(1).start()); Assert.assertTrue(seqTasks.get(2).start()); Assert.assertTrue(seqTasks.get(3).start()); - Assert.assertEquals(8, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(8, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } /** @@ -1133,7 +1149,7 @@ public void testSelectFileBaseOnDirtyDataRateWithHeavySelect2Aligned() Assert.assertTrue(seqTasks.get(0).start()); Assert.assertTrue(seqTasks.get(1).start()); Assert.assertTrue(seqTasks.get(2).start()); - Assert.assertEquals(6, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(6, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } // endregion diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionTaskTest.java index 43ab0a8ad6032..72ccd6921be6c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleCompactionTaskTest.java @@ -186,8 +186,8 @@ public void settleWithOnlyAllDirtyFilesByMods() Assert.assertEquals(TsFileResourceStatus.DELETED, tsFileResource.getStatus()); } - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(0, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); DataNodeTTLCache.getInstance().clearAllTTLForTree(); validateTargetDatas(sourceDatas, Collections.emptyList()); @@ -225,8 +225,8 @@ public void settleWithOnlyPartialDirtyFilesByMods() Assert.assertEquals(TsFileResourceStatus.DELETED, tsFileResource.getStatus()); } - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); DataNodeTTLCache.getInstance().clearAllTTLForTree(); validateTargetDatas(sourceDatas, Collections.emptyList()); @@ -258,8 +258,8 @@ public void settleWithMixedDirtyFilesByMods() 0, tsFileManager, allDeletedFiles, partialDeletedFiles, false, getPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(6, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(6, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); DataNodeTTLCache.getInstance().clearAllTTLForTree(); validateTargetDatas(sourceDatas, Collections.emptyList()); @@ -270,8 +270,8 @@ public void settleWithMixedDirtyFilesByMods() 0, tsFileManager, allDeletedFiles, partialDeletedFiles, true, getPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); for (TsFileResource tsFileResource : seqResources) { Assert.assertEquals(TsFileResourceStatus.DELETED, tsFileResource.getStatus()); } @@ -312,8 +312,8 @@ public void settleWithOnlyAllDirtyFilesByTTL() Assert.assertEquals(TsFileResourceStatus.DELETED, tsFileResource.getStatus()); } - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(0, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); DataNodeTTLCache.getInstance().clearAllTTLForTree(); validateTargetDatas(sourceDatas, Collections.emptyList()); @@ -354,8 +354,8 @@ public void settleWithOnlyAllDirtyFilesByTTL2() Assert.assertEquals(TsFileResourceStatus.DELETED, tsFileResource.getStatus()); } - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(0, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); DataNodeTTLCache.getInstance().clearAllTTLForTree(); validateTargetDatas(sourceDatas, Collections.emptyList()); @@ -401,8 +401,8 @@ public void settleWithOnlyPartialDirtyFilesByTTL() Assert.assertEquals(TsFileResourceStatus.DELETED, tsFileResource.getStatus()); } - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); DataNodeTTLCache.getInstance().clearAllTTLForTree(); validateTargetDatas(sourceDatas, Collections.emptyList()); @@ -427,7 +427,7 @@ public void getModsFileAfterSettleCompaction() getPerformer(), 0); Assert.assertTrue(task.start()); - List tsFileList = tsFileManager.getTsFileList(true); + List tsFileList = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); for (TsFileResource resource : tsFileList) { Assert.assertTrue( resource @@ -475,8 +475,8 @@ public void settleWithMixedDirtyFilesByTTL() Assert.assertEquals(TsFileResourceStatus.DELETED, tsFileResource.getStatus()); } - Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); - Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); + Assert.assertEquals(1, tsFileManager.getTsFileList(false, COMPACTION_TEST_SG).size()); DataNodeTTLCache.getInstance().clearAllTTLForTree(); validateTargetDatas(sourceDatas, Collections.emptyList()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java index 2de0d95bbc98c..9df6bcef7708e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java @@ -132,14 +132,19 @@ public void testCompactionWithAllNullRows1() throws IOException { InnerSpaceCompactionTask task = new InnerSpaceCompactionTask(0, tsFileManager, seqResources, true, getPerformer(), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(10, target.getFileStartTime()); Assert.assertEquals(12, target.getFileEndTime()); InnerSpaceCompactionTask task2 = new InnerSpaceCompactionTask( - 0, tsFileManager, tsFileManager.getTsFileList(true), true, getPerformer(), 0); + 0, + tsFileManager, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + true, + getPerformer(), + 0); Assert.assertTrue(task2.start()); - TsFileResource target2 = tsFileManager.getTsFileList(true).get(0); + TsFileResource target2 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(10, target2.getFileStartTime()); Assert.assertEquals(12, target2.getFileEndTime()); } @@ -167,14 +172,19 @@ public void testCompactionWithAllNullRows2() throws IOException { InnerSpaceCompactionTask task = new InnerSpaceCompactionTask(0, tsFileManager, seqResources, true, getPerformer(), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(1, target.getFileStartTime()); Assert.assertEquals(12, target.getFileEndTime()); InnerSpaceCompactionTask task2 = new InnerSpaceCompactionTask( - 0, tsFileManager, tsFileManager.getTsFileList(true), true, getPerformer(), 0); + 0, + tsFileManager, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + true, + getPerformer(), + 0); Assert.assertTrue(task2.start()); - TsFileResource target2 = tsFileManager.getTsFileList(true).get(0); + TsFileResource target2 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(1, target2.getFileStartTime()); Assert.assertEquals(12, target2.getFileEndTime()); } @@ -200,14 +210,19 @@ public void testCompactionWithAllNullRows3() throws IOException { InnerSpaceCompactionTask task = new InnerSpaceCompactionTask(0, tsFileManager, seqResources, true, getPerformer(), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(10000, target.getFileStartTime()); Assert.assertEquals(19999, target.getFileEndTime()); InnerSpaceCompactionTask task2 = new InnerSpaceCompactionTask( - 0, tsFileManager, tsFileManager.getTsFileList(true), true, getPerformer(), 0); + 0, + tsFileManager, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + true, + getPerformer(), + 0); Assert.assertTrue(task2.start()); - TsFileResource target2 = tsFileManager.getTsFileList(true).get(0); + TsFileResource target2 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(10000, target2.getFileStartTime()); Assert.assertEquals(19999, target2.getFileEndTime()); } @@ -233,14 +248,19 @@ public void testCompactionWithAllNullRows4() throws IOException { InnerSpaceCompactionTask task = new InnerSpaceCompactionTask(0, tsFileManager, seqResources, true, getPerformer(), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(100000, target.getFileStartTime()); Assert.assertEquals(199999, target.getFileEndTime()); InnerSpaceCompactionTask task2 = new InnerSpaceCompactionTask( - 0, tsFileManager, tsFileManager.getTsFileList(true), true, getPerformer(), 0); + 0, + tsFileManager, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + true, + getPerformer(), + 0); Assert.assertTrue(task2.start()); - TsFileResource target2 = tsFileManager.getTsFileList(true).get(0); + TsFileResource target2 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(100000, target2.getFileStartTime()); Assert.assertEquals(199999, target2.getFileEndTime()); } @@ -273,7 +293,7 @@ public void testCompactionWithAllDeletion1() throws IOException { InnerSpaceCompactionTask task = new InnerSpaceCompactionTask(0, tsFileManager, seqResources, true, getPerformer(), 0); Assert.assertTrue(task.start()); - Assert.assertTrue(tsFileManager.getTsFileList(true).isEmpty()); + Assert.assertTrue(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).isEmpty()); } @Test @@ -304,7 +324,7 @@ public void testCompactionWithAllDeletion2() throws IOException { InnerSpaceCompactionTask task = new InnerSpaceCompactionTask(0, tsFileManager, seqResources, true, getPerformer(), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(12, target.getFileStartTime()); Assert.assertEquals(12, target.getFileEndTime()); } @@ -367,7 +387,7 @@ public void testCompactionWithAllValueColumnDeletion() throws IOException, Illeg InnerSpaceCompactionTask task = new InnerSpaceCompactionTask(0, tsFileManager, seqResources, true, getPerformer(), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(10, target.getFileStartTime()); Assert.assertEquals(12, target.getFileEndTime()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java index 877756b73d5b7..65b4eb1eb86ac 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java @@ -101,7 +101,7 @@ public void testAllDataExpired() throws IOException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - Assert.assertEquals(0, tsFileManager.getTsFileList(true).size()); + Assert.assertEquals(0, tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).size()); } @Test @@ -130,7 +130,7 @@ public void testPartialDataExpired() throws IOException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertTrue(target.getFileStartTime() > startTime && target.getFileEndTime() == endTime); } @@ -159,7 +159,7 @@ public void testTableNotExist() throws IOException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, getPerformer(performerType), 0); Assert.assertTrue(task.start()); - TsFileResource target = tsFileManager.getTsFileList(true).get(0); + TsFileResource target = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertTrue(target.getFileStartTime() == startTime && target.getFileEndTime() == endTime); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java index 8096003b9c916..716d01b14b99f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java @@ -128,12 +128,12 @@ public void testCrossSpaceCompactionOfTwoTableModelWithFastCompactionPerformer() 0, 0); Assert.assertTrue(task.start()); - TsFileResource targetResource0 = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetResource0 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource0.getTsFile().getAbsolutePath())) { Assert.assertEquals(1, reader.getTableSchemaMap().size()); } - TsFileResource targetResource1 = tsFileManager.getTsFileList(true).get(1); + TsFileResource targetResource1 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource1.getTsFile().getAbsolutePath())) { Assert.assertEquals(1, reader.getTableSchemaMap().size()); @@ -173,7 +173,11 @@ public void testCompactionWithMultiTableSchema() throws IOException { Assert.assertTrue(task.start()); try (TsFileSequenceReader reader = new TsFileSequenceReader( - tsFileManager.getTsFileList(true).get(0).getTsFile().getAbsolutePath())) { + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getTsFile() + .getAbsolutePath())) { Assert.assertEquals(1, reader.getTableSchemaMap().size()); } } @@ -234,7 +238,7 @@ public void testCrossSpaceCompactionOfTableModelCanNotMatchTableSchema() throws 0, 0); Assert.assertTrue(task.start()); - TsFileResource targetResource = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetResource = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(1, targetResource.getDevices().size()); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource.getTsFile().getAbsolutePath())) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java index 06fff63881799..9c2543a9159af 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java @@ -105,7 +105,7 @@ public void testSequenceInnerSpaceCompactionOfTwoTableModel() throws IOException new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - TsFileResource targetResource = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetResource = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource.getTsFile().getAbsolutePath())) { TableSchema tableSchema = reader.getTableSchemaMap().get("t1"); @@ -167,7 +167,7 @@ public void testSequenceInnerSpaceCompactionOfTableModelCanNotMatchTableSchema() new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); - TsFileResource targetResource = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetResource = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); Assert.assertEquals(1, targetResource.getDevices().size()); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource.getTsFile().getAbsolutePath())) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java index 54882c30285b5..18c9a7b238363 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java @@ -152,12 +152,12 @@ public void testCrossSpaceCompactionOfTwoTableModelWithReadPointCompactionPerfor 0); Assert.assertTrue(task.start()); // Assert can not pass for now - TsFileResource targetResource0 = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetResource0 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource0.getTsFile().getAbsolutePath())) { Assert.assertEquals(1, reader.getTableSchemaMap().size()); } - TsFileResource targetResource1 = tsFileManager.getTsFileList(true).get(1); + TsFileResource targetResource1 = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(1); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource1.getTsFile().getAbsolutePath())) { Assert.assertEquals(1, reader.getTableSchemaMap().size()); @@ -197,7 +197,11 @@ public void testCompactionWithMultiTableSchema() throws IOException { Assert.assertTrue(task.start()); try (TsFileSequenceReader reader = new TsFileSequenceReader( - tsFileManager.getTsFileList(true).get(0).getTsFile().getAbsolutePath())) { + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .getTsFile() + .getAbsolutePath())) { Assert.assertEquals(1, reader.getTableSchemaMap().size()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskMemCostEstimatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskMemCostEstimatorTest.java index e96ea2653d890..8f9fa2f8c5592 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskMemCostEstimatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskMemCostEstimatorTest.java @@ -70,7 +70,7 @@ public void testEstimateReadChunkInnerSpaceCompactionTaskMemCost() throws IOException, MetadataException, WriteProcessException { createFiles(3, 10, 5, 100000, 0, 0, 50, 50, true, true); tsFileManager.addAll(seqResources, true); - List tsFileList = tsFileManager.getTsFileList(true); + List tsFileList = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); System.out.println(tsFileList.get(0).getTsFile().getAbsolutePath()); long cost = new ReadChunkInnerCompactionEstimator().estimateInnerCompactionMemory(tsFileList); Assert.assertTrue(cost > 0); @@ -81,7 +81,7 @@ public void testEstimateReadChunkInnerSpaceCompactionTaskMemCost2() throws IOException, MetadataException, WriteProcessException { createFiles(3, 10, 5, 100, 0, 0, 50, 50, false, true); tsFileManager.addAll(seqResources, true); - List tsFileList = tsFileManager.getTsFileList(true); + List tsFileList = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); long cost = new ReadChunkInnerCompactionEstimator().estimateInnerCompactionMemory(tsFileList); Assert.assertTrue(cost > 0); } @@ -103,7 +103,7 @@ public void testEstimateFastCompactionInnerSpaceCompactionTaskMemCost2() throws IOException, MetadataException, WriteProcessException { createFiles(3, 10, 5, 100, 0, 0, 50, 50, false, true); tsFileManager.addAll(seqResources, true); - List tsFileList = tsFileManager.getTsFileList(true); + List tsFileList = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG); long cost = new FastCompactionInnerCompactionEstimator().estimateInnerCompactionMemory(tsFileList); Assert.assertTrue(cost > 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MultiTsFileDeviceIteratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MultiTsFileDeviceIteratorTest.java index b5010e60449df..e899f111c095b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MultiTsFileDeviceIteratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MultiTsFileDeviceIteratorTest.java @@ -147,7 +147,7 @@ public void testMeasurementIterator() throws IOException, MetadataException { new InnerSpaceCompactionTask( 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0) .start(); - TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0); + TsFileResource targetFile = tsFileManager.getTsFileList(true, COMPACTION_TEST_SG).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetFile.getTsFile().getAbsolutePath())) { Assert.assertEquals(4000, reader.getAllMeasurements().size()); @@ -450,7 +450,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer() int deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -490,7 +490,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer() new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadChunkCompactionPerformer(), 0L); @@ -507,7 +507,10 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer() } } generateModsFile( - seriesPaths, tsFileManager.getTsFileList(true), Long.MIN_VALUE, Long.MAX_VALUE); + seriesPaths, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Long.MIN_VALUE, + Long.MAX_VALUE); deleteTimeseriesInMManager(seriesPaths); @@ -529,7 +532,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer() deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -541,17 +544,24 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer() TsFileGeneratorUtils.alignDeviceOffset = oldAlignedDeviceOffset; List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources( - tsFileManager.getTsFileList(true), true); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true); ReadChunkCompactionPerformer performer = - new ReadChunkCompactionPerformer(tsFileManager.getTsFileList(true), targetResources.get(0)); + new ReadChunkCompactionPerformer( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), targetResources.get(0)); performer.setSummary(new CompactionTaskSummary()); performer.perform(); CompactionUtils.moveTargetFile( targetResources, CompactionTaskType.INNER_SEQ, COMPACTION_TEST_SG); tsFileManager.replace( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0); - tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources, + 0); + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .setStatusForTest(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -605,7 +615,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer2() int deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -645,7 +655,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer2() new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadChunkCompactionPerformer(), 0L); @@ -662,7 +672,10 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer2() } } generateModsFile( - seriesPaths, tsFileManager.getTsFileList(true), Long.MIN_VALUE, Long.MAX_VALUE); + seriesPaths, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Long.MIN_VALUE, + Long.MAX_VALUE); deleteTimeseriesInMManager(seriesPaths); @@ -685,7 +698,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer2() deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -697,17 +710,24 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadChunkPerformer2() TsFileGeneratorUtils.alignDeviceOffset = oldAlignedDeviceOffset; List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources( - tsFileManager.getTsFileList(true), true); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true); ReadChunkCompactionPerformer performer = - new ReadChunkCompactionPerformer(tsFileManager.getTsFileList(true), targetResources.get(0)); + new ReadChunkCompactionPerformer( + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), targetResources.get(0)); performer.setSummary(new CompactionTaskSummary()); performer.perform(); CompactionUtils.moveTargetFile( targetResources, CompactionTaskType.INNER_SEQ, COMPACTION_TEST_SG); tsFileManager.replace( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0); - tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources, + 0); + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .setStatusForTest(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -755,7 +775,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer() thro int deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -795,7 +815,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer() thro new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0L); @@ -812,7 +832,10 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer() thro } } generateModsFile( - seriesPaths, tsFileManager.getTsFileList(true), Long.MIN_VALUE, Long.MAX_VALUE); + seriesPaths, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Long.MIN_VALUE, + Long.MAX_VALUE); deleteTimeseriesInMManager(seriesPaths); @@ -834,7 +857,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer() thro deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -846,18 +869,26 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer() thro TsFileGeneratorUtils.alignDeviceOffset = oldAlignedDeviceOffset; List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources( - tsFileManager.getTsFileList(true), true); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true); ReadPointCompactionPerformer performer = new ReadPointCompactionPerformer( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources); performer.setSummary(new CompactionTaskSummary()); performer.perform(); CompactionUtils.moveTargetFile( targetResources, CompactionTaskType.INNER_SEQ, COMPACTION_TEST_SG); tsFileManager.replace( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0); - tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources, + 0); + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .setStatusForTest(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -905,7 +936,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer2() thr int deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -945,7 +976,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer2() thr new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new ReadPointCompactionPerformer(), 0L); @@ -962,7 +993,10 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer2() thr } } generateModsFile( - seriesPaths, tsFileManager.getTsFileList(true), Long.MIN_VALUE, Long.MAX_VALUE); + seriesPaths, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Long.MIN_VALUE, + Long.MAX_VALUE); deleteTimeseriesInMManager(seriesPaths); @@ -994,7 +1028,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer2() thr deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -1006,18 +1040,26 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByReadPointPerformer2() thr TsFileGeneratorUtils.alignDeviceOffset = oldAlignedDeviceOffset; List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources( - tsFileManager.getTsFileList(true), true); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true); ReadPointCompactionPerformer performer = new ReadPointCompactionPerformer( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources); performer.setSummary(new CompactionTaskSummary()); performer.perform(); CompactionUtils.moveTargetFile( targetResources, CompactionTaskType.INNER_SEQ, COMPACTION_TEST_SG); tsFileManager.replace( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0); - tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources, + 0); + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .setStatusForTest(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -1065,7 +1107,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer() throws Ex int deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -1105,7 +1147,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer() throws Ex new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0L); @@ -1122,7 +1164,10 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer() throws Ex } } generateModsFile( - seriesPaths, tsFileManager.getTsFileList(true), Long.MIN_VALUE, Long.MAX_VALUE); + seriesPaths, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Long.MIN_VALUE, + Long.MAX_VALUE); deleteTimeseriesInMManager(seriesPaths); @@ -1144,7 +1189,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer() throws Ex deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -1156,18 +1201,26 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer() throws Ex TsFileGeneratorUtils.alignDeviceOffset = oldAlignedDeviceOffset; List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources( - tsFileManager.getTsFileList(true), true); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true); FastCompactionPerformer performer = new FastCompactionPerformer( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources); performer.setSummary(new FastCompactionTaskSummary()); performer.perform(); CompactionUtils.moveTargetFile( targetResources, CompactionTaskType.INNER_SEQ, COMPACTION_TEST_SG); tsFileManager.replace( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0); - tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources, + 0); + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .setStatusForTest(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -1215,7 +1268,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer2() throws E int deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -1255,7 +1308,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer2() throws E new InnerSpaceCompactionTask( 0, tsFileManager, - tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true, new FastCompactionPerformer(false), 0L); @@ -1272,7 +1325,10 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer2() throws E } } generateModsFile( - seriesPaths, tsFileManager.getTsFileList(true), Long.MIN_VALUE, Long.MAX_VALUE); + seriesPaths, + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Long.MIN_VALUE, + Long.MAX_VALUE); deleteTimeseriesInMManager(seriesPaths); @@ -1304,7 +1360,7 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer2() throws E deviceNum = 0; try (MultiTsFileDeviceIterator multiTsFileDeviceIterator = - new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true))) { + new MultiTsFileDeviceIterator(tsFileManager.getTsFileList(true, COMPACTION_TEST_SG))) { while (multiTsFileDeviceIterator.hasNextDevice()) { Pair deviceInfo = multiTsFileDeviceIterator.nextDevice(); Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left); @@ -1316,18 +1372,26 @@ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer2() throws E TsFileGeneratorUtils.alignDeviceOffset = oldAlignedDeviceOffset; List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources( - tsFileManager.getTsFileList(true), true); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), true); FastCompactionPerformer performer = new FastCompactionPerformer( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources); performer.setSummary(new FastCompactionTaskSummary()); performer.perform(); CompactionUtils.moveTargetFile( targetResources, CompactionTaskType.INNER_SEQ, COMPACTION_TEST_SG); tsFileManager.replace( - tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0); - tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL); + tsFileManager.getTsFileList(true, COMPACTION_TEST_SG), + Collections.emptyList(), + targetResources, + 0); + tsFileManager + .getTsFileList(true, COMPACTION_TEST_SG) + .get(0) + .setStatusForTest(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java index 84d8deac3d78e..199e6b460d187 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java @@ -231,9 +231,10 @@ public void testLoadSnapshot() new SnapshotLoader(snapshotDir.getAbsolutePath(), testSgName, "0") .loadSnapshotForStateMachine(); Assert.assertNotNull(dataRegion); - List resource = dataRegion.getTsFileManager().getTsFileList(true); - assertEquals(100, resource.size()); - assertEquals( + List resource = + dataRegion.getTsFileManager().getTsFileList(true, testSgName); + Assert.assertEquals(100, resource.size()); + Assert.assertEquals( new Pair<>(100L, 100L), CompressionRatio.getInstance().getDataRegionRatioMap().get("0")); } finally { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkDataDirectWriteTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkDataDirectWriteTest.java index 8db10eaf1b574..537cc6b58cde7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkDataDirectWriteTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkDataDirectWriteTest.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.queryengine.plan.scheduler.load.ChunkOffsetCalculator; import org.apache.iotdb.db.storageengine.load.splitter.ChunkData.ChunkLayout; @@ -204,6 +205,56 @@ public void testAlignedChunkEndBuildsOnlyCurrentPhysicalChunk() throws Exception // Time Partition Filtering Tests // ------------------------------------------------------------------------- + // Points outside the current time partition are dropped from a non-aligned chunk + @Test + public void testDatabasePartitionBoundsSurviveChunkEncodingAndSerialization() throws Exception { + String database = "root.custom_partition"; + TimePartitionUtils.updateDatabaseTimePartitionConfig( + database, new TDatabaseSchema().setTimePartitionOrigin(50).setTimePartitionInterval(100)); + try { + long[] times = {150, 249, 250}; + for (boolean aligned : new boolean[] {false, true}) { + ChunkData chunkData = + ChunkData.createChunkData( + aligned, + DEFAULT_DEVICE, + aligned ? createTimeChunkHeader() : createChunkHeader("s1", TSDataType.INT32), + new TTimePartitionSlot(150), + database); + chunkData.writeDecodePage(times, aligned ? new Object[3] : new Object[] {1, 2, 3}, 2); + chunkData.endChunk(); + assertEquals(2, chunkData.getChunks().get(0).getChunkStatistic().getCount()); + assertEquals(249, chunkData.getChunks().get(0).getChunkStatistic().getEndTime()); + + if (aligned) { + BatchedAlignedValueChunkData values = + new BatchedAlignedValueChunkData((AlignedChunkData) chunkData); + values.addValueChunk(createChunkHeader("s1", TSDataType.INT32)); + values.writeDecodeValuePage( + times, + new TsPrimitiveType[] { + TsPrimitiveType.getByType(TSDataType.INT32, 1), + TsPrimitiveType.getByType(TSDataType.INT32, 2), + TsPrimitiveType.getByType(TSDataType.INT32, 3) + }, + TSDataType.INT32); + values.endChunk(); + assertEquals(2, values.getChunks().get(0).getChunkStatistic().getCount()); + assertEquals(249, values.getChunks().get(0).getChunkStatistic().getEndTime()); + } + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + chunkData.serialize(new DataOutputStream(bytes)); + ChunkData deserialized = + (ChunkData) TsFileData.deserialize(new ByteArrayInputStream(bytes.toByteArray())); + assertEquals(new TTimePartitionSlot(150), deserialized.getTimePartitionSlot()); + assertEquals(2, deserialized.getChunks().get(0).getChunkStatistic().getCount()); + } + } finally { + TimePartitionUtils.removeDatabaseTimePartitionConfig(database); + } + } + // Points outside the current time partition are dropped from a non-aligned chunk @Test public void testNonAlignedChunkSelectsPointsFromCurrentTimePartition() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/AbstractConfigNodeClient.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/AbstractConfigNodeClient.java index 2d20166fbb56c..f0f4d491e81ef 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/AbstractConfigNodeClient.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/AbstractConfigNodeClient.java @@ -153,7 +153,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TSetDataReplicationFactorReq; import org.apache.iotdb.confignode.rpc.thrift.TSetSchemaReplicationFactorReq; import org.apache.iotdb.confignode.rpc.thrift.TSetSchemaTemplateReq; -import org.apache.iotdb.confignode.rpc.thrift.TSetTimePartitionIntervalReq; import org.apache.iotdb.confignode.rpc.thrift.TShowAINodesResp; import org.apache.iotdb.confignode.rpc.thrift.TShowCQResp; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; @@ -593,11 +592,6 @@ public TSStatus setDataReplicationFactor(TSetDataReplicationFactorReq req) throw return null; } - @Override - public TSStatus setTimePartitionInterval(TSetTimePartitionIntervalReq req) throws TException { - return null; - } - @Override public TCountDatabaseResp countMatchedDatabases(TGetDatabaseReq req) throws TException { return null; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java index 51d1ee3a705fd..b2a76030a2d4f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java @@ -110,7 +110,10 @@ public List> getTimePartitionRange( dataPartitionMap.get(storageGroup).get(seriesPartitionSlot); List timePartitionSlotList = map.keySet().stream() - .filter(key -> TimePartitionUtils.satisfyPartitionStartTime(timeFilter, key.startTime)) + .filter( + key -> + TimePartitionUtils.satisfyPartitionStartTime( + timeFilter, key.startTime, storageGroup)) .sorted(Comparator.comparingLong(TTimePartitionSlot::getStartTime)) .collect(toList()); @@ -153,7 +156,8 @@ public List getDataRegionReplicaSetWithTimeFilter( return regionReplicaSetMap.entrySet().stream() .filter( entry -> - TimePartitionUtils.satisfyPartitionStartTime(timeFilter, entry.getKey().startTime)) + TimePartitionUtils.satisfyPartitionStartTime( + timeFilter, entry.getKey().startTime, storageGroup)) .flatMap(entry -> entry.getValue().stream()) .distinct() .collect(toList()); @@ -176,7 +180,8 @@ public List getDataRegionReplicaSetWithTimeFilter( return dataPartitionMap.get(database).get(seriesPartitionSlot).entrySet().stream() .filter( entry -> - TimePartitionUtils.satisfyPartitionStartTime(timeFilter, entry.getKey().startTime)) + TimePartitionUtils.satisfyPartitionStartTime( + timeFilter, entry.getKey().startTime, database)) .flatMap(entry -> entry.getValue().stream()) .distinct() .collect(toList()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionQueryParam.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionQueryParam.java index 4282fefade1dc..0fc9266637534 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionQueryParam.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionQueryParam.java @@ -31,6 +31,7 @@ public class DataPartitionQueryParam { private String databaseName; private IDeviceID deviceID; private List timePartitionSlotList = new ArrayList<>(); + private List timeList = new ArrayList<>(); // it will be set to true in query when there exist filter like: time <= XXX // (-oo, timePartitionSlotList.get(0)) @@ -75,6 +76,18 @@ public void setTimePartitionSlotList(List timePartitionSlotL this.timePartitionSlotList = timePartitionSlotList; } + public List getTimeList() { + return timeList; + } + + public void setTimeList(List timeList) { + this.timeList = timeList; + } + + public boolean hasTimeList() { + return timeList != null && !timeList.isEmpty(); + } + public boolean isNeedLeftAll() { return needLeftAll; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionTable.java index d154f1813e1b7..c70a22615e3f3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionTable.java @@ -273,12 +273,12 @@ public Map getLastDataAllotTable() { * @param currentTimeSlot The current TimeSlot */ public Set autoCleanPartitionTable( - long TTL, TTimePartitionSlot currentTimeSlot) { + long TTL, TTimePartitionSlot currentTimeSlot, String database) { Set removedTimePartitionSlots = new HashSet<>(); dataPartitionMap.forEach( (seriesPartitionSlot, seriesPartitionTable) -> removedTimePartitionSlots.addAll( - seriesPartitionTable.autoCleanPartitionTable(TTL, currentTimeSlot))); + seriesPartitionTable.autoCleanPartitionTable(TTL, currentTimeSlot, database))); return removedTimePartitionSlots; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java index 4c448d916194a..eba58da9c9296 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java @@ -263,10 +263,22 @@ public TConsensusGroupId getLastConsensusGroupId() { */ public List autoCleanPartitionTable( long TTL, TTimePartitionSlot currentTimeSlot) { - final long timePartitionInterval = + return cleanExpiredPartitions( + TTL, + currentTimeSlot, CommonDateTimeUtils.convertMilliTimeWithPrecision( TimePartitionUtils.getTimePartitionInterval(), - CommonDescriptor.getInstance().getConfig().getTimestampPrecision()); + CommonDescriptor.getInstance().getConfig().getTimestampPrecision())); + } + + public List autoCleanPartitionTable( + long TTL, TTimePartitionSlot currentTimeSlot, String database) { + return cleanExpiredPartitions( + TTL, currentTimeSlot, TimePartitionUtils.getTimePartitionInterval(database)); + } + + private List cleanExpiredPartitions( + long TTL, TTimePartitionSlot currentTimeSlot, long timePartitionInterval) { List removedTimePartitions = new ArrayList<>(); Iterator>> iterator = seriesPartitionMap.entrySet().iterator(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeId.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeId.java index 8bfdf2979f6fc..adc842baafff9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeId.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeId.java @@ -38,6 +38,10 @@ public PlanNodeId(String id) { this.id = id; } + public PlanNodeId(String id, String database) { + this.id = database != null ? database + "_" + id : id; + } + public String getId() { return this.id; } @@ -68,6 +72,10 @@ public static PlanNodeId deserialize(InputStream stream) throws IOException { return new PlanNodeId(ReadWriteIOUtils.readString(stream)); } + public static PlanNodeId deserialize(InputStream stream, String database) throws IOException { + return new PlanNodeId(ReadWriteIOUtils.readString(stream), database); + } + public void serialize(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(id, byteBuffer); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index d50be22154eb4..cc86cc691673f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -281,6 +281,7 @@ private ColumnHeaderConstant() { public static final String SCHEMA_REPLICATION_FACTOR_TABLE_MODEL = "schema_replication_factor"; public static final String DATA_REPLICATION_FACTOR_TABLE_MODEL = "data_replication_factor"; + public static final String TIME_PARTITION_ORIGIN_TABLE_MODEL = "time_partition_origin"; public static final String TIME_PARTITION_INTERVAL_TABLE_MODEL = "time_partition_interval"; public static final String SCHEMA_REGION_GROUP_NUM_TABLE_MODEL = "schema_region_group_num"; public static final String MAX_SCHEMA_REGION_GROUP_NUM_TABLE_MODEL = @@ -856,6 +857,7 @@ private ColumnHeaderConstant() { new ColumnHeader(COLUMN_TTL, TSDataType.TEXT), new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32), new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32), + new ColumnHeader(TIME_PARTITION_ORIGIN, TSDataType.INT64), new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64), new ColumnHeader(SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(MAX_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 3efaf7e1a9a34..7589e8be2ef5a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -106,6 +106,9 @@ public class InformationSchema { databaseTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.DATA_REPLICATION_FACTOR_TABLE_MODEL, TSDataType.INT32)); + databaseTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.TIME_PARTITION_ORIGIN_TABLE_MODEL, TSDataType.INT64)); databaseTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.TIME_PARTITION_INTERVAL_TABLE_MODEL, TSDataType.INT64)); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java index 8ae40176831a3..10acebb4b48c4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java @@ -42,6 +42,7 @@ import org.apache.thrift.transport.TTransportException; import java.io.DataOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -227,7 +228,7 @@ public static TTimeSlotList deserializeTTimePartitionSlotList(ByteBuffer buffer) } public static void serializeTTimePartitionSlot( - TTimePartitionSlot timePartitionSlot, DataOutputStream stream) { + TTimePartitionSlot timePartitionSlot, DataOutputStream stream) throws IOException { try { timePartitionSlot.write(generateWriteProtocol(stream)); } catch (TException e) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java index 8083e3db1c87a..372d0974dac82 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java @@ -20,210 +20,395 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.tsfile.read.filter.basic.Filter; import java.math.BigInteger; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import static com.google.common.math.LongMath.saturatedAdd; import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.saturateToLong; public class TimePartitionUtils { - private static final BigInteger BIG_LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); private static final BigInteger BIG_LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); private static final BigInteger BIG_ONE = BigInteger.ONE; - /** - * Time partition origin for dividing database, the time unit is the same with IoTDB's - * TimestampPrecision - */ - private static volatile long timePartitionOrigin = - CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); - - /** Time range for dividing database, the time unit is the same with IoTDB's TimestampPrecision */ - private static volatile long timePartitionInterval = - CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); - - private static volatile long timePartitionLowerBoundWithoutOverflow; - private static volatile long timePartitionUpperBoundWithoutOverflow; - private static volatile boolean timePartitionLowerBoundOverflow; - private static volatile boolean timePartitionUpperBoundOverflow; + private static volatile DatabaseTimePartitionConfig defaultConfig = + new DatabaseTimePartitionConfig( + CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(), + CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()); + private static final Map databaseConfigCache = + new ConcurrentHashMap<>(); + + // Update or add database-specific time partition configuration + public static void updateDatabaseTimePartitionConfig(String database, TDatabaseSchema schema) { + String timestampPrecision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + long interval = + CommonDateTimeUtils.convertMilliTimeWithPrecision( + schema.isSetTimePartitionInterval() + ? schema.getTimePartitionInterval() + : CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(), + timestampPrecision); + long origin = + CommonDateTimeUtils.convertMilliTimeWithPrecision( + schema.isSetTimePartitionOrigin() + ? schema.getTimePartitionOrigin() + : CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(), + timestampPrecision); + databaseConfigCache.put(database, new DatabaseTimePartitionConfig(origin, interval)); + } - static { - updateTimePartitionBound(); + public static void updateDatabaseTimePartitionConfigs(Map schemaMap) { + if (schemaMap == null || schemaMap.isEmpty()) { + return; + } + schemaMap.forEach( + (database, schema) -> { + if (database != null && schema != null) { + updateDatabaseTimePartitionConfig(database, schema); + } + }); } - private static void updateTimePartitionBound() { - BigInteger minPartition = getTimePartitionIdAsBigInteger(Long.MIN_VALUE); - BigInteger maxPartition = getTimePartitionIdAsBigInteger(Long.MAX_VALUE); - timePartitionLowerBoundOverflow = minPartition.compareTo(BIG_LONG_MIN) < 0; - timePartitionUpperBoundOverflow = maxPartition.compareTo(BIG_LONG_MAX) > 0; + // Remove database-specific time partition configuration + public static void removeDatabaseTimePartitionConfig(String database) { + databaseConfigCache.remove(database); + } - // The lower/upper bounds are the starts of the first and last regular slots. If the exact - // partition id at a timestamp is outside the long id range, the corresponding boundary slot - // is represented by the nearest long id and starts at the timestamp limit. - BigInteger firstRepresentablePartition = minPartition.max(BIG_LONG_MIN); - BigInteger firstRepresentableStart = - getTimePartitionStartTimeAsBigInteger(firstRepresentablePartition); - if (timePartitionLowerBoundOverflow || firstRepresentableStart.compareTo(BIG_LONG_MIN) < 0) { - firstRepresentableStart = - firstRepresentableStart.add(BigInteger.valueOf(timePartitionInterval)); - } - timePartitionLowerBoundWithoutOverflow = saturateToLong(firstRepresentableStart); + public static void clearDatabaseTimePartitionConfigCache() { + databaseConfigCache.clear(); + } - BigInteger lastRepresentablePartition = maxPartition.min(BIG_LONG_MAX); - BigInteger lastRepresentableStart; - if (timePartitionUpperBoundOverflow) { - lastRepresentableStart = - getTimePartitionStartTimeAsBigInteger(lastRepresentablePartition.add(BIG_ONE)); - } else { - lastRepresentableStart = getTimePartitionStartTimeAsBigInteger(lastRepresentablePartition); - } - timePartitionUpperBoundWithoutOverflow = saturateToLong(lastRepresentableStart); + public static boolean hasDatabaseTimePartitionConfig(String database) { + return database != null && databaseConfigCache.containsKey(database); } public static TTimePartitionSlot getTimePartitionSlot(long time) { - TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot(); - timePartitionSlot.setStartTime(getTimePartitionLowerBound(time)); - return timePartitionSlot; + return getTimePartitionSlot(time, null); + } + + public static TTimePartitionSlot getTimePartitionSlot(long time, String database) { + return getDatabaseConfig(database).getTimePartitionSlot(time); } public static long getTimePartitionInterval() { - return timePartitionInterval; + return getTimePartitionInterval(null); + } + + public static long getTimePartitionInterval(String database) { + return getDatabaseConfig(database).getTimePartitionInterval(); } public static long getTimePartitionLowerBound(long time) { - if (time < timePartitionLowerBoundWithoutOverflow) { - return Long.MIN_VALUE; - } - if (time >= timePartitionUpperBoundWithoutOverflow) { - return timePartitionUpperBoundWithoutOverflow; - } - return getTimePartitionStartTime(getTimePartitionId(time)); + return getTimePartitionLowerBound(time, null); + } + + public static long getTimePartitionLowerBound(long time, String database) { + return getDatabaseConfig(database).getTimePartitionLowerBound(time); } public static long getTimePartitionUpperBound(long time) { - if (time >= timePartitionUpperBoundWithoutOverflow) { - return Long.MAX_VALUE; - } - if (time < timePartitionLowerBoundWithoutOverflow) { - return timePartitionLowerBoundWithoutOverflow; - } - return saturatedAdd(getTimePartitionLowerBound(time), timePartitionInterval); + return getTimePartitionUpperBound(time, null); + } + + public static long getTimePartitionUpperBound(long time, String database) { + return getDatabaseConfig(database).getTimePartitionUpperBound(time); } public static long getTimePartitionEndTime(long time) { - long upperBound = getTimePartitionUpperBound(time); - if (upperBound != Long.MAX_VALUE) { - return upperBound - 1; - } - return getTimePartitionLowerBound(time) == getTimePartitionLowerBound(Long.MAX_VALUE) - ? Long.MAX_VALUE - : Long.MAX_VALUE - 1; + return getTimePartitionEndTime(time, null); + } + + public static long getTimePartitionEndTime(long time, String database) { + return getDatabaseConfig(database).getTimePartitionEndTime(time); } public static boolean isAfterOrEqualToTimePartitionUpperBound( long time, long timePartitionStartTime, long timePartitionUpperBound) { - if (timePartitionUpperBound != Long.MAX_VALUE) { - return time >= timePartitionUpperBound; - } - return time == Long.MAX_VALUE && getTimePartitionLowerBound(time) != timePartitionStartTime; + return isAfterOrEqualToTimePartitionUpperBound( + time, timePartitionStartTime, timePartitionUpperBound, null); + } + + public static boolean isAfterOrEqualToTimePartitionUpperBound( + long time, long timePartitionStartTime, long timePartitionUpperBound, String database) { + return getDatabaseConfig(database) + .isAfterOrEqualToTimePartitionUpperBound( + time, timePartitionStartTime, timePartitionUpperBound); } public static boolean isTimePartitionStartTime(long time) { - return getTimePartitionLowerBound(time) == time; + return isTimePartitionStartTime(time, null); + } + + public static boolean isTimePartitionStartTime(long time, String database) { + return getDatabaseConfig(database).isTimePartitionStartTime(time); } public static long getTimePartitionId(long time) { - final long timeFromOrigin; - try { - timeFromOrigin = Math.subtractExact(time, timePartitionOrigin); - } catch (ArithmeticException e) { - return getTimePartitionIdWithoutOverflow(time); - } - return Math.floorDiv(timeFromOrigin, timePartitionInterval); + return getTimePartitionId(time, null); + } + + public static long getTimePartitionId(long time, String database) { + return getDatabaseConfig(database).getTimePartitionId(time); } public static long getTimePartitionIdWithoutOverflow(long time) { - BigInteger partitionId = getTimePartitionIdAsBigInteger(time); - if (partitionId.compareTo(BIG_LONG_MIN) < 0) { - return Long.MIN_VALUE; - } - if (partitionId.compareTo(BIG_LONG_MAX) > 0) { - return Long.MAX_VALUE; - } - return partitionId.longValue(); + return getTimePartitionIdWithoutOverflow(time, null); } - private static BigInteger getTimePartitionIdAsBigInteger(long time) { - BigInteger bigTime = BigInteger.valueOf(time).subtract(BigInteger.valueOf(timePartitionOrigin)); - BigInteger bigTimePartitionInterval = BigInteger.valueOf(timePartitionInterval); - return bigTime.compareTo(BigInteger.ZERO) > 0 - || bigTime.remainder(bigTimePartitionInterval).equals(BigInteger.ZERO) - ? bigTime.divide(bigTimePartitionInterval) - : bigTime.divide(bigTimePartitionInterval).subtract(BigInteger.ONE); + public static long getTimePartitionIdWithoutOverflow(long time, String database) { + return getDatabaseConfig(database).getTimePartitionIdWithoutOverflow(time); } public static long getStartTimeByPartitionId(long partitionId) { - return getTimePartitionStartTime(partitionId); + return getStartTimeByPartitionId(partitionId, null); + } + + public static long getStartTimeByPartitionId(long partitionId, String database) { + return getDatabaseConfig(database).getStartTimeByPartitionId(partitionId); } public static boolean satisfyPartitionId(long startTime, long endTime, long partitionId) { - long startPartition = getTimePartitionId(startTime); - long endPartition = getTimePartitionId(endTime); - return startPartition <= partitionId && endPartition >= partitionId; + return satisfyPartitionId(startTime, endTime, partitionId, null); + } + + public static boolean satisfyPartitionId( + long startTime, long endTime, long partitionId, String database) { + return getDatabaseConfig(database).satisfyPartitionId(startTime, endTime, partitionId); } public static boolean satisfyPartitionStartTime(Filter timeFilter, long partitionStartTime) { - if (timeFilter == null) { - return true; - } - long partitionEndTime = getTimePartitionEndTime(partitionStartTime); - return timeFilter.satisfyStartEndTime(partitionStartTime, partitionEndTime); + return satisfyPartitionStartTime(timeFilter, partitionStartTime, null); + } + + public static boolean satisfyPartitionStartTime( + Filter timeFilter, long partitionStartTime, String database) { + return getDatabaseConfig(database).satisfyPartitionStartTime(timeFilter, partitionStartTime); } public static boolean satisfyTimePartition(Filter timeFilter, long partitionId) { - return satisfyPartitionStartTime(timeFilter, getTimePartitionStartTime(partitionId)); + return satisfyTimePartition(timeFilter, partitionId, null); } - private static long getTimePartitionStartTime(long partitionId) { - // A clamped boundary id represents the saturated underflow/overflow partition, whose start is - // the corresponding timestamp boundary rather than the mathematical start of that id. - if (partitionId == Long.MIN_VALUE && timePartitionLowerBoundOverflow) { - return Long.MIN_VALUE; - } - if (partitionId == Long.MAX_VALUE && timePartitionUpperBoundOverflow) { - return timePartitionUpperBoundWithoutOverflow; - } - return saturateToLong(getTimePartitionStartTimeAsBigInteger(BigInteger.valueOf(partitionId))); + public static boolean satisfyTimePartition(Filter timeFilter, long partitionId, String database) { + return getDatabaseConfig(database).satisfyTimePartition(timeFilter, partitionId); } - private static BigInteger getTimePartitionStartTimeAsBigInteger(BigInteger partitionId) { - return partitionId - .multiply(BigInteger.valueOf(timePartitionInterval)) - .add(BigInteger.valueOf(timePartitionOrigin)); + public static long getEstimateTimePartitionSize(long startTime, long endTime) { + return getEstimateTimePartitionSize(startTime, endTime, null); } - public static void setTimePartitionInterval(long timePartitionInterval) { - TimePartitionUtils.timePartitionInterval = timePartitionInterval; - updateTimePartitionBound(); + public static long getEstimateTimePartitionSize(long startTime, long endTime, String database) { + return getDatabaseConfig(database).getEstimateTimePartitionSize(startTime, endTime); } - public static void setTimePartitionOrigin(long timePartitionOrigin) { - TimePartitionUtils.timePartitionOrigin = timePartitionOrigin; - updateTimePartitionBound(); + private static DatabaseTimePartitionConfig getDatabaseConfig(String database) { + DatabaseTimePartitionConfig config = + database == null ? null : databaseConfigCache.get(database); + return config == null ? defaultConfig : config; } - public static long getEstimateTimePartitionSize(long startTime, long endTime) { - BigInteger estimateSize = - BigInteger.valueOf(endTime) - .subtract(BigInteger.valueOf(startTime)) - .divide(BigInteger.valueOf(timePartitionInterval)) - .add(BigInteger.ONE); - if (estimateSize.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { - return Long.MAX_VALUE; - } - return estimateSize.longValue(); + public static long getTimePartitionOrigin() { + return defaultConfig.getTimePartitionOrigin(); + } + + public static long getTimePartitionOrigin(String database) { + return getDatabaseConfig(database).getTimePartitionOrigin(); + } + + public static synchronized void setTimePartitionOrigin(long origin) { + defaultConfig = + new DatabaseTimePartitionConfig(origin, defaultConfig.getTimePartitionInterval()); + } + + public static synchronized void setTimePartitionInterval(long interval) { + defaultConfig = + new DatabaseTimePartitionConfig(defaultConfig.getTimePartitionOrigin(), interval); + } + + public static class DatabaseTimePartitionConfig { + private final long timePartitionOrigin; + private final long timePartitionInterval; + private long timePartitionLowerBoundWithoutOverflow; + private long timePartitionUpperBoundWithoutOverflow; + private boolean timePartitionLowerBoundOverflow; + private boolean timePartitionUpperBoundOverflow; + + public DatabaseTimePartitionConfig(long origin, long interval) { + this.timePartitionOrigin = origin; + this.timePartitionInterval = interval; + updateTimePartitionBound(); + } + + public long getTimePartitionOrigin() { + return timePartitionOrigin; + } + + private void updateTimePartitionBound() { + BigInteger minPartition = getTimePartitionIdAsBigInteger(Long.MIN_VALUE); + BigInteger maxPartition = getTimePartitionIdAsBigInteger(Long.MAX_VALUE); + timePartitionLowerBoundOverflow = minPartition.compareTo(BIG_LONG_MIN) < 0; + timePartitionUpperBoundOverflow = maxPartition.compareTo(BIG_LONG_MAX) > 0; + + // The lower/upper bounds are the starts of the first and last regular slots. If the exact + // partition id at a timestamp is outside the long id range, the corresponding boundary slot + // is represented by the nearest long id and starts at the timestamp limit. + BigInteger firstRepresentablePartition = minPartition.max(BIG_LONG_MIN); + BigInteger firstRepresentableStart = + getTimePartitionStartTimeAsBigInteger(firstRepresentablePartition); + if (timePartitionLowerBoundOverflow || firstRepresentableStart.compareTo(BIG_LONG_MIN) < 0) { + firstRepresentableStart = + firstRepresentableStart.add(BigInteger.valueOf(timePartitionInterval)); + } + timePartitionLowerBoundWithoutOverflow = saturateToLong(firstRepresentableStart); + + BigInteger lastRepresentablePartition = maxPartition.min(BIG_LONG_MAX); + BigInteger lastRepresentableStart; + if (timePartitionUpperBoundOverflow) { + lastRepresentableStart = + getTimePartitionStartTimeAsBigInteger(lastRepresentablePartition.add(BIG_ONE)); + } else { + lastRepresentableStart = getTimePartitionStartTimeAsBigInteger(lastRepresentablePartition); + } + timePartitionUpperBoundWithoutOverflow = saturateToLong(lastRepresentableStart); + } + + public TTimePartitionSlot getTimePartitionSlot(long time) { + TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot(); + timePartitionSlot.setStartTime(getTimePartitionLowerBound(time)); + return timePartitionSlot; + } + + public long getTimePartitionInterval() { + return timePartitionInterval; + } + + public long getTimePartitionLowerBound(long time) { + if (time < timePartitionLowerBoundWithoutOverflow) { + return Long.MIN_VALUE; + } + if (time >= timePartitionUpperBoundWithoutOverflow) { + return timePartitionUpperBoundWithoutOverflow; + } + return getTimePartitionStartTime(getTimePartitionId(time)); + } + + public long getTimePartitionUpperBound(long time) { + if (time >= timePartitionUpperBoundWithoutOverflow) { + return Long.MAX_VALUE; + } + if (time < timePartitionLowerBoundWithoutOverflow) { + return timePartitionLowerBoundWithoutOverflow; + } + return saturatedAdd(getTimePartitionLowerBound(time), timePartitionInterval); + } + + public long getTimePartitionEndTime(long time) { + long upperBound = getTimePartitionUpperBound(time); + if (upperBound != Long.MAX_VALUE) { + return upperBound - 1; + } + return getTimePartitionLowerBound(time) == getTimePartitionLowerBound(Long.MAX_VALUE) + ? Long.MAX_VALUE + : Long.MAX_VALUE - 1; + } + + public boolean isAfterOrEqualToTimePartitionUpperBound( + long time, long timePartitionStartTime, long timePartitionUpperBound) { + if (timePartitionUpperBound != Long.MAX_VALUE) { + return time >= timePartitionUpperBound; + } + return time == Long.MAX_VALUE && getTimePartitionLowerBound(time) != timePartitionStartTime; + } + + public boolean isTimePartitionStartTime(long time) { + return getTimePartitionLowerBound(time) == time; + } + + public long getTimePartitionId(long time) { + final long timeFromOrigin; + try { + timeFromOrigin = Math.subtractExact(time, timePartitionOrigin); + } catch (ArithmeticException e) { + return getTimePartitionIdWithoutOverflow(time); + } + return Math.floorDiv(timeFromOrigin, timePartitionInterval); + } + + public long getTimePartitionIdWithoutOverflow(long time) { + BigInteger partitionId = getTimePartitionIdAsBigInteger(time); + if (partitionId.compareTo(BIG_LONG_MIN) < 0) { + return Long.MIN_VALUE; + } + if (partitionId.compareTo(BIG_LONG_MAX) > 0) { + return Long.MAX_VALUE; + } + return partitionId.longValue(); + } + + private BigInteger getTimePartitionIdAsBigInteger(long time) { + BigInteger bigTime = + BigInteger.valueOf(time).subtract(BigInteger.valueOf(timePartitionOrigin)); + BigInteger bigTimePartitionInterval = BigInteger.valueOf(timePartitionInterval); + return bigTime.compareTo(BigInteger.ZERO) > 0 + || bigTime.remainder(bigTimePartitionInterval).equals(BigInteger.ZERO) + ? bigTime.divide(bigTimePartitionInterval) + : bigTime.divide(bigTimePartitionInterval).subtract(BigInteger.ONE); + } + + public long getStartTimeByPartitionId(long partitionId) { + return getTimePartitionStartTime(partitionId); + } + + public boolean satisfyPartitionId(long startTime, long endTime, long partitionId) { + long startPartition = getTimePartitionId(startTime); + long endPartition = getTimePartitionId(endTime); + return startPartition <= partitionId && endPartition >= partitionId; + } + + public boolean satisfyPartitionStartTime(Filter timeFilter, long partitionStartTime) { + if (timeFilter == null) { + return true; + } + long partitionEndTime = getTimePartitionEndTime(partitionStartTime); + return timeFilter.satisfyStartEndTime(partitionStartTime, partitionEndTime); + } + + public boolean satisfyTimePartition(Filter timeFilter, long partitionId) { + return satisfyPartitionStartTime(timeFilter, getTimePartitionStartTime(partitionId)); + } + + private long getTimePartitionStartTime(long partitionId) { + // A clamped boundary id represents the saturated underflow/overflow partition, whose start is + // the corresponding timestamp boundary rather than the mathematical start of that id. + if (partitionId == Long.MIN_VALUE && timePartitionLowerBoundOverflow) { + return Long.MIN_VALUE; + } + if (partitionId == Long.MAX_VALUE && timePartitionUpperBoundOverflow) { + return timePartitionUpperBoundWithoutOverflow; + } + return saturateToLong(getTimePartitionStartTimeAsBigInteger(BigInteger.valueOf(partitionId))); + } + + private BigInteger getTimePartitionStartTimeAsBigInteger(BigInteger partitionId) { + return partitionId + .multiply(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.valueOf(timePartitionOrigin)); + } + + public long getEstimateTimePartitionSize(long startTime, long endTime) { + BigInteger estimateSize = + BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .divide(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.ONE); + if (estimateSize.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + return Long.MAX_VALUE; + } + return estimateSize.longValue(); + } } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java index 9f1b3f327fbe3..6e0fefbf59587 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.tsfile.read.filter.factory.TimeFilterApi; import org.junit.After; @@ -29,6 +30,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; public class TimePartitionUtilsTest { @@ -40,6 +42,7 @@ public class TimePartitionUtilsTest { @Before public void setUp() { + CommonDescriptor.getInstance().getConfig().setTimestampPrecision("ms"); previousTimePartitionOrigin = CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); previousTimePartitionInterval = @@ -48,12 +51,15 @@ public void setUp() { CommonDescriptor.getInstance() .getConfig() .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL); + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); TimePartitionUtils.setTimePartitionOrigin(TEST_TIME_PARTITION_ORIGIN); TimePartitionUtils.setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL); } @After public void tearDown() { + CommonDescriptor.getInstance().getConfig().setTimestampPrecision("ms"); + TimePartitionUtils.clearDatabaseTimePartitionConfigCache(); CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(previousTimePartitionOrigin); CommonDescriptor.getInstance() .getConfig() @@ -68,7 +74,7 @@ public void testGetTimePartitionSlot_StartOfInterval() { TTimePartitionSlot expectedSlot = new TTimePartitionSlot(); expectedSlot.setStartTime(TEST_TIME_PARTITION_ORIGIN); - TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime); + TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "global"); assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); } @@ -78,7 +84,7 @@ public void testGetTimePartitionSlot_MiddleOfInterval() { TTimePartitionSlot expectedSlot = new TTimePartitionSlot(); expectedSlot.setStartTime(TEST_TIME_PARTITION_ORIGIN); - TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime); + TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "global"); assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); } @@ -88,7 +94,7 @@ public void testGetTimePartitionSlot_EndOfInterval() { TTimePartitionSlot expectedSlot = new TTimePartitionSlot(); expectedSlot.setStartTime(TEST_TIME_PARTITION_ORIGIN); - TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime); + TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "global"); assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); } @@ -98,7 +104,7 @@ public void testGetTimePartitionSlot_NegativeTime() { TTimePartitionSlot expectedSlot = new TTimePartitionSlot(); expectedSlot.setStartTime(TEST_TIME_PARTITION_ORIGIN - TEST_TIME_PARTITION_INTERVAL); - TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime); + TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "global"); assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); } @@ -108,25 +114,210 @@ public void testGetTimePartitionSlot_NegativeBoundaryTime() { TTimePartitionSlot expectedSlot = new TTimePartitionSlot(); expectedSlot.setStartTime(TEST_TIME_PARTITION_ORIGIN - TEST_TIME_PARTITION_INTERVAL); - TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime); + TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "global"); assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); } @Test public void testOverflow() { long testTime = Long.MIN_VALUE; - TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime); + TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "global"); Assert.assertTrue(actualSlot.getStartTime() < 0); testTime += 1; - long lowerBound = TimePartitionUtils.getTimePartitionLowerBound(testTime); + long lowerBound = TimePartitionUtils.getTimePartitionLowerBound(testTime, "global"); assertEquals(Long.MIN_VALUE, lowerBound); testTime = Long.MAX_VALUE; - actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime); + actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "global"); Assert.assertTrue(actualSlot.getStartTime() > 0); - long upperBound = TimePartitionUtils.getTimePartitionUpperBound(testTime); + long upperBound = TimePartitionUtils.getTimePartitionUpperBound(testTime, "global"); assertEquals(Long.MAX_VALUE, upperBound); } + @Test + public void testDatabasePartitionBoundsAtTimestampLimits() { + String database = "boundary.db"; + TDatabaseSchema schema = + new TDatabaseSchema().setTimePartitionOrigin(1).setTimePartitionInterval(1); + TimePartitionUtils.updateDatabaseTimePartitionConfig(database, schema); + + assertEquals(Long.MIN_VALUE, TimePartitionUtils.getTimePartitionId(Long.MIN_VALUE, database)); + assertEquals( + Long.MIN_VALUE, + TimePartitionUtils.getTimePartitionSlot(Long.MIN_VALUE + 1, database).getStartTime()); + assertEquals( + Long.MIN_VALUE + 2, + TimePartitionUtils.getTimePartitionUpperBound(Long.MIN_VALUE, database)); + assertEquals( + Long.MIN_VALUE + 1, TimePartitionUtils.getTimePartitionEndTime(Long.MIN_VALUE, database)); + assertFalse( + TimePartitionUtils.satisfyTimePartition( + TimeFilterApi.eq(Long.MIN_VALUE + 2), Long.MIN_VALUE, database)); + + schema.setTimePartitionOrigin(0); + TimePartitionUtils.updateDatabaseTimePartitionConfig(database, schema); + assertEquals( + Long.MAX_VALUE - 1, + TimePartitionUtils.getTimePartitionEndTime(Long.MAX_VALUE - 1, database)); + assertEquals( + Long.MAX_VALUE, TimePartitionUtils.getTimePartitionEndTime(Long.MAX_VALUE, database)); + Assert.assertTrue( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + Long.MAX_VALUE, Long.MAX_VALUE - 1, Long.MAX_VALUE, database)); + assertFalse( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, database)); + assertEquals( + Long.MAX_VALUE, + TimePartitionUtils.getEstimateTimePartitionSize(Long.MIN_VALUE, Long.MAX_VALUE, database)); + assertEquals(TEST_TIME_PARTITION_INTERVAL, TimePartitionUtils.getTimePartitionInterval()); + } + + @Test + public void testDatabaseLevelTimePartitionIdDoesNotOverflow() { + TDatabaseSchema schema = new TDatabaseSchema(); + schema.setName("overflow.db"); + schema.setTimePartitionOrigin(1000L); + schema.setTimePartitionInterval(100L); + TimePartitionUtils.updateDatabaseTimePartitionConfig("overflow.db", schema); + + assertEquals( + -92233720368547769L, TimePartitionUtils.getTimePartitionId(Long.MIN_VALUE, "overflow.db")); + } + + @Test + public void testDatabaseLevelTimePartitionFilterPruning() { + TDatabaseSchema schema = new TDatabaseSchema(); + schema.setName("filter.db"); + schema.setTimePartitionOrigin(1000L); + schema.setTimePartitionInterval(100L); + TimePartitionUtils.updateDatabaseTimePartitionConfig("filter.db", schema); + + assertFalse(TimePartitionUtils.satisfyTimePartition(TimeFilterApi.eq(1200L), 1L, "filter.db")); + assertFalse( + TimePartitionUtils.satisfyTimePartition( + TimeFilterApi.eq(TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL), 0L)); + } + + @Test + public void testDatabaseLevelTimePartition() { + // Create a database schema with custom time partition settings + TDatabaseSchema schema = new TDatabaseSchema(); + schema.setName("test.db"); + schema.setTimePartitionInterval(7200000L); // 2 hours + schema.setTimePartitionOrigin(2000L); + + // Update database configuration + TimePartitionUtils.updateDatabaseTimePartitionConfig("test.db", schema); + + // Test with custom database settings + long testTime = 2000L; + TTimePartitionSlot expectedSlot = new TTimePartitionSlot(); + expectedSlot.setStartTime(2000L); + + TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "test.db"); + assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); + + // Test with different time in the same partition + testTime = 2000L + 3600000L; // 1 hour later + actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "test.db"); + assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); + + // Test with time in next partition + testTime = 2000L + 7200000L; // 2 hours later + expectedSlot.setStartTime(2000L + 7200000L); + actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "test.db"); + assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); + } + + @Test + public void testDatabaseLevelTimePartitionUsesTimestampPrecision() { + CommonDescriptor.getInstance().getConfig().setTimestampPrecision("ns"); + TDatabaseSchema schema = new TDatabaseSchema(); + schema.setName("test.db"); + schema.setTimePartitionInterval(7200000L); + schema.setTimePartitionOrigin(2000L); + + TimePartitionUtils.updateDatabaseTimePartitionConfig("test.db", schema); + + assertEquals(7200000_000_000L, TimePartitionUtils.getTimePartitionInterval("test.db")); + assertEquals(2000_000_000L, TimePartitionUtils.getTimePartitionOrigin("test.db")); + assertEquals( + 2000_000_000L, + TimePartitionUtils.getTimePartitionSlot(2000_000_000L + 1, "test.db").getStartTime()); + } + + @Test + public void testDatabaseLevelTimePartitionDefaultUsesTimestampPrecision() { + CommonDescriptor.getInstance().getConfig().setTimestampPrecision("us"); + TDatabaseSchema schema = new TDatabaseSchema(); + schema.setName("test.db"); + + TimePartitionUtils.updateDatabaseTimePartitionConfig("test.db", schema); + + assertEquals( + TEST_TIME_PARTITION_INTERVAL * 1000L, + TimePartitionUtils.getTimePartitionInterval("test.db")); + assertEquals( + TEST_TIME_PARTITION_ORIGIN * 1000L, TimePartitionUtils.getTimePartitionOrigin("test.db")); + assertEquals( + TEST_TIME_PARTITION_ORIGIN * 1000L, + TimePartitionUtils.getTimePartitionSlot(TEST_TIME_PARTITION_ORIGIN * 1000L + 1, "test.db") + .getStartTime()); + } + + @Test + public void testDatabaseLevelTimePartitionFallbackToGlobal() { + // Test with database that doesn't have custom settings + long testTime = TEST_TIME_PARTITION_ORIGIN; + TTimePartitionSlot expectedSlot = new TTimePartitionSlot(); + expectedSlot.setStartTime(TEST_TIME_PARTITION_ORIGIN); + + TTimePartitionSlot actualSlot = + TimePartitionUtils.getTimePartitionSlot(testTime, "nonexistent.db"); + assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); + } + + @Test + public void testHasDatabaseTimePartitionConfig() { + TDatabaseSchema schema = new TDatabaseSchema(); + schema.setName("test.db"); + schema.setTimePartitionInterval(7200000L); + schema.setTimePartitionOrigin(2000L); + + Assert.assertFalse(TimePartitionUtils.hasDatabaseTimePartitionConfig(null)); + Assert.assertFalse(TimePartitionUtils.hasDatabaseTimePartitionConfig("test.db")); + + TimePartitionUtils.updateDatabaseTimePartitionConfig("test.db", schema); + Assert.assertTrue(TimePartitionUtils.hasDatabaseTimePartitionConfig("test.db")); + + TimePartitionUtils.removeDatabaseTimePartitionConfig("test.db"); + Assert.assertFalse(TimePartitionUtils.hasDatabaseTimePartitionConfig("test.db")); + } + + @Test + public void testRemoveDatabaseTimePartitionConfig() { + // Create and update database configuration + TDatabaseSchema schema = new TDatabaseSchema(); + schema.setName("test.db"); + schema.setTimePartitionInterval(7200000L); + schema.setTimePartitionOrigin(2000L); + TimePartitionUtils.updateDatabaseTimePartitionConfig("test.db", schema); + + // Verify custom configuration is used + long testTime = 2000L; + TTimePartitionSlot actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "test.db"); + assertEquals(2000L, actualSlot.getStartTime()); + + // Remove database configuration + TimePartitionUtils.removeDatabaseTimePartitionConfig("test.db"); + + // Verify fallback to global settings + TTimePartitionSlot expectedSlot = new TTimePartitionSlot(); + expectedSlot.setStartTime(TEST_TIME_PARTITION_ORIGIN); + actualSlot = TimePartitionUtils.getTimePartitionSlot(testTime, "test.db"); + assertEquals(expectedSlot.getStartTime(), actualSlot.getStartTime()); + } + @Test public void testIsTimePartitionStartTimeWithOrigin() { Assert.assertTrue(TimePartitionUtils.isTimePartitionStartTime(TEST_TIME_PARTITION_ORIGIN)); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 0b733829d87b1..6294d2ca2bd96 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -207,11 +207,6 @@ struct TSetDataReplicationFactorReq { 2: required i32 dataReplicationFactor } -struct TSetTimePartitionIntervalReq { - 1: required string database - 2: required i64 timePartitionInterval -} - struct TCountDatabaseResp { 1: required common.TSStatus status 2: optional i32 count @@ -1514,9 +1509,6 @@ service IConfigNodeRPCService { /** Update the specific Database's DataReplicationFactor */ common.TSStatus setDataReplicationFactor(TSetDataReplicationFactorReq req) - /** Update the specific Database's PartitionInterval */ - common.TSStatus setTimePartitionInterval(TSetTimePartitionIntervalReq req) - /** Count the matched Databases */ TCountDatabaseResp countMatchedDatabases(TGetDatabaseReq req)