Skip to content

Commit 2b9fdb9

Browse files
authored
[to dev/1.3] Supports streaming load mem chunk & Push down filter, limit and offset to MemPointIterator
1 parent 38cc43e commit 2b9fdb9

25 files changed

Lines changed: 3088 additions & 431 deletions
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.it.aligned;
21+
22+
import org.apache.iotdb.it.env.EnvFactory;
23+
24+
import org.junit.After;
25+
import org.junit.Before;
26+
import org.junit.Test;
27+
28+
import java.sql.Connection;
29+
import java.sql.ResultSet;
30+
import java.sql.SQLException;
31+
import java.sql.Statement;
32+
import java.util.Locale;
33+
34+
import static org.junit.Assert.assertEquals;
35+
36+
public class IoTDBAlignedMemQueryIT {
37+
38+
@Before
39+
public void setUp() throws Exception {
40+
Locale.setDefault(Locale.ENGLISH);
41+
EnvFactory.getEnv()
42+
.getConfig()
43+
.getCommonConfig()
44+
.setPartitionInterval(1000)
45+
.setMemtableSizeThreshold(10000);
46+
EnvFactory.getEnv().initClusterEnvironment();
47+
}
48+
49+
@After
50+
public void tearDown() throws Exception {
51+
EnvFactory.getEnv().cleanClusterEnvironment();
52+
}
53+
54+
@Test
55+
public void test() throws SQLException {
56+
try (Connection connection = EnvFactory.getEnv().getConnection();
57+
Statement statement = connection.createStatement()) {
58+
59+
statement.execute("insert into root.vehicle.d0(time,s0) aligned values (10,310)");
60+
statement.execute("insert into root.vehicle.d0(time,s3) aligned values (10,'text')");
61+
statement.execute("insert into root.vehicle.d0(time,s4) aligned values (10,true)");
62+
63+
try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) {
64+
int cnt = 0;
65+
while (set.next()) {
66+
cnt++;
67+
}
68+
assertEquals(1, cnt);
69+
}
70+
71+
statement.execute("insert into root.vehicle.d1(time,s0,s1) aligned values (1,1,1)");
72+
statement.execute("insert into root.vehicle.d1(time,s0) aligned values (2,2)");
73+
statement.execute("insert into root.vehicle.d1(time,s1) aligned values (3,3)");
74+
75+
try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d1")) {
76+
int cnt = 0;
77+
while (set.next()) {
78+
cnt++;
79+
}
80+
assertEquals(2, cnt);
81+
}
82+
statement.execute("flush");
83+
84+
statement.execute("insert into root.vehicle.d1(time,s0,s1) aligned values (1,1,1)");
85+
statement.execute("insert into root.vehicle.d1(time,s0) aligned values (2,2)");
86+
statement.execute("insert into root.vehicle.d1(time,s1) aligned values (3,3)");
87+
88+
try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d1")) {
89+
int cnt = 0;
90+
while (set.next()) {
91+
cnt++;
92+
}
93+
assertEquals(2, cnt);
94+
}
95+
}
96+
}
97+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,9 @@ public class IoTDBConfig {
439439
*/
440440
private int tvListSortThreshold = 0;
441441

442+
/** Enable streaming query mem chunk */
443+
private boolean streamingQueryMemChunk = true;
444+
442445
/** Enable inner space compaction for sequence files */
443446
private volatile boolean enableSeqSpaceCompaction = true;
444447

@@ -2319,6 +2322,14 @@ public void setTVListSortThreshold(int tvListSortThreshold) {
23192322
this.tvListSortThreshold = tvListSortThreshold;
23202323
}
23212324

2325+
public boolean isStreamingQueryMemChunk() {
2326+
return streamingQueryMemChunk;
2327+
}
2328+
2329+
public void setStreamingQueryMemChunk(boolean streamingQueryMemChunk) {
2330+
this.streamingQueryMemChunk = streamingQueryMemChunk;
2331+
}
2332+
23222333
public boolean isRpcThriftCompressionEnable() {
23232334
return rpcThriftCompressionEnable;
23242335
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,8 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
127127
} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
128128
loadFromMem = true;
129129

130-
timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
130+
timeSeriesMetadata =
131+
(TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath, globalTimeFilter);
131132
if (timeSeriesMetadata != null) {
132133
timeSeriesMetadata.setChunkMetadataLoader(
133134
new MemChunkMetadataLoader(resource, seriesPath, context, globalTimeFilter));
@@ -192,7 +193,8 @@ public static AlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
192193
} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
193194
loadFromMem = true;
194195
alignedTimeSeriesMetadata =
195-
(AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(alignedPath);
196+
(AlignedTimeSeriesMetadata)
197+
resource.getTimeSeriesMetadata(alignedPath, globalTimeFilter);
196198
if (alignedTimeSeriesMetadata != null) {
197199
alignedTimeSeriesMetadata.setChunkMetadataLoader(
198200
new MemAlignedChunkMetadataLoader(resource, alignedPath, context, globalTimeFilter));

0 commit comments

Comments
 (0)