Skip to content

Commit dfcd489

Browse files
authored
[Performance] Pre-size collections with known cardinality (#18854)
Backport #18851 by re-scanning dev/1.3 for equivalent collection materialization paths and adapting capacity hints to this branch's schema, plan, Session, Pipe, memtable, and UDF implementations. (cherry picked from commit a78f5a7)
1 parent c056ce6 commit dfcd489

80 files changed

Lines changed: 206 additions & 170 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,8 @@ public IoTDBJDBCDataSet(
113113
this.zoneId = zoneId;
114114
columnSize = columnNameList.size();
115115

116-
this.columnNameList = new ArrayList<>();
117-
this.columnTypeList = new ArrayList<>();
116+
this.columnNameList = new ArrayList<>(columnSize + (ignoreTimeStamp ? 0 : 1));
117+
this.columnTypeList = new ArrayList<>(columnSize + (ignoreTimeStamp ? 0 : 1));
118118
if (!ignoreTimeStamp) {
119119
this.columnNameList.add(TIMESTAMP_STR);
120120
this.columnTypeList.add(String.valueOf(TSDataType.INT64));
@@ -222,8 +222,8 @@ public IoTDBJDBCDataSet(
222222
this.zoneId = zoneId;
223223
columnSize = columnNameList.size();
224224

225-
this.columnNameList = new ArrayList<>();
226-
this.columnTypeList = new ArrayList<>();
225+
this.columnNameList = new ArrayList<>(columnSize + (ignoreTimeStamp ? 0 : 1));
226+
this.columnTypeList = new ArrayList<>(columnSize + (ignoreTimeStamp ? 0 : 1));
227227
if (!ignoreTimeStamp) {
228228
this.columnNameList.add(TIMESTAMP_STR);
229229
this.columnTypeList.add(String.valueOf(TSDataType.INT64));

‎iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ public IoTDBRpcDataSet(
111111
this.moreData = moreData;
112112
columnSize = columnNameList.size();
113113

114-
this.columnNameList = new ArrayList<>();
115-
this.columnTypeList = new ArrayList<>();
114+
this.columnNameList = new ArrayList<>(columnSize + (ignoreTimeStamp ? 0 : 1));
115+
this.columnTypeList = new ArrayList<>(columnSize + (ignoreTimeStamp ? 0 : 1));
116116
if (!ignoreTimeStamp) {
117117
this.columnNameList.add(TIMESTAMP_STR);
118118
this.columnTypeList.add(String.valueOf(TSDataType.INT64));
@@ -205,8 +205,8 @@ public IoTDBRpcDataSet(
205205
this.moreData = moreData;
206206
columnSize = columnNameList.size();
207207

208-
this.columnNameList = new ArrayList<>();
209-
this.columnTypeList = new ArrayList<>();
208+
this.columnNameList = new ArrayList<>(columnSize + (ignoreTimeStamp ? 0 : 1));
209+
this.columnTypeList = new ArrayList<>(columnSize + (ignoreTimeStamp ? 0 : 1));
210210
if (!ignoreTimeStamp) {
211211
this.columnNameList.add(TIMESTAMP_STR);
212212
this.columnTypeList.add(String.valueOf(TSDataType.INT64));

‎iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2531,7 +2531,7 @@ private List<ByteBuffer> objectValuesListToByteBufferList(
25312531
List<List<TSDataType>> typesList,
25322532
List<List<String>> measurementsList)
25332533
throws IoTDBConnectionException {
2534-
List<ByteBuffer> buffersList = new ArrayList<>();
2534+
List<ByteBuffer> buffersList = new ArrayList<>(valuesList.size());
25352535
for (int i = 0; i < valuesList.size(); i++) {
25362536
ByteBuffer buffer =
25372537
SessionUtils.getValueBuffer(typesList.get(i), valuesList.get(i), measurementsList.get(i));
@@ -2870,8 +2870,8 @@ private void updateTSInsertTabletsReq(
28702870
sortTablet(tablet);
28712871
}
28722872
request.addToPrefixPaths(tablet.deviceId);
2873-
List<String> measurements = new ArrayList<>();
2874-
List<Integer> dataTypes = new ArrayList<>();
2873+
List<String> measurements = new ArrayList<>(tablet.getSchemas().size());
2874+
List<Integer> dataTypes = new ArrayList<>(tablet.getSchemas().size());
28752875
request.setIsAligned(isAligned);
28762876
for (IMeasurementSchema measurementSchema : tablet.getSchemas()) {
28772877
if (measurementSchema.getMeasurementId() == null) {
@@ -3530,9 +3530,9 @@ public void createSchemaTemplate(
35303530
List<List<TSEncoding>> encodings,
35313531
List<CompressionType> compressors)
35323532
throws IoTDBConnectionException, StatementExecutionException {
3533-
List<String> flatMeasurements = new ArrayList<>();
3534-
List<TSDataType> flatDataTypes = new ArrayList<>();
3535-
List<TSEncoding> flatEncodings = new ArrayList<>();
3533+
List<String> flatMeasurements = new ArrayList<>(measurements.size());
3534+
List<TSDataType> flatDataTypes = new ArrayList<>(measurements.size());
3535+
List<TSEncoding> flatEncodings = new ArrayList<>(measurements.size());
35363536
for (int idx = 0; idx < measurements.size(); idx++) {
35373537
flatMeasurements.add(measurements.get(idx).get(0));
35383538
flatDataTypes.add(dataTypes.get(idx).get(0));

‎iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ public static List<TEndPoint> parseSeedNodeUrls(List<String> nodeUrls) {
330330
if (nodeUrls == null) {
331331
throw new NumberFormatException("nodeUrls is null");
332332
}
333-
List<TEndPoint> endPointsList = new ArrayList<>();
333+
List<TEndPoint> endPointsList = new ArrayList<>(nodeUrls.size());
334334
for (String nodeUrl : nodeUrls) {
335335
TEndPoint endPoint = UrlUtils.parseTEndPointIpv4AndIpv6Url(nodeUrl);
336336
endPointsList.add(endPoint);

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorPlan.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,14 @@
2626
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
2727
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
2828

29+
import com.google.common.collect.Sets;
2930
import org.apache.tsfile.utils.ReadWriteIOUtils;
3031
import org.slf4j.Logger;
3132

3233
import java.io.DataOutputStream;
3334
import java.io.IOException;
3435
import java.nio.ByteBuffer;
3536
import java.util.ArrayList;
36-
import java.util.HashSet;
3737
import java.util.List;
3838
import java.util.Objects;
3939
import java.util.Set;
@@ -181,7 +181,7 @@ protected void deserializeImpl(ByteBuffer buffer) {
181181
this.permissions = null;
182182
} else {
183183
final int permissionsSize = buffer.getInt();
184-
this.permissions = new HashSet<>();
184+
this.permissions = Sets.newHashSetWithExpectedSize(permissionsSize);
185185
for (int i = 0; i < permissionsSize; i++) {
186186
permissions.add(buffer.getInt());
187187
}

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ protected void deserializeImpl(ByteBuffer buffer) throws IOException {
8888
this.modelName = ReadWriteIOUtils.readString(buffer);
8989
this.modelInformation = ModelInformation.deserialize(buffer);
9090
int size = ReadWriteIOUtils.readInt(buffer);
91-
this.nodeIds = new ArrayList<>();
91+
this.nodeIds = new ArrayList<>(size);
9292
for (int i = 0; i < size; i++) {
9393
this.nodeIds.add(ReadWriteIOUtils.readInt(buffer));
9494
}

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,13 @@
2525
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
2626
import org.apache.iotdb.db.schemaengine.template.Template;
2727

28+
import com.google.common.collect.Maps;
2829
import org.apache.tsfile.utils.ReadWriteIOUtils;
2930

3031
import java.io.DataOutputStream;
3132
import java.io.IOException;
3233
import java.nio.ByteBuffer;
3334
import java.util.ArrayList;
34-
import java.util.HashMap;
3535
import java.util.List;
3636
import java.util.Map;
3737
import java.util.Objects;
@@ -69,7 +69,7 @@ protected void serializeImpl(DataOutputStream stream) throws IOException {
6969
@Override
7070
protected void deserializeImpl(ByteBuffer buffer) throws IOException {
7171
int size = ReadWriteIOUtils.readInt(buffer);
72-
templateSetInfo = new HashMap<>();
72+
templateSetInfo = Maps.newHashMapWithExpectedSize(size);
7373
for (int i = 0; i < size; i++) {
7474
PartialPath pattern = (PartialPath) PathDeserializeUtil.deserialize(buffer);
7575
int templateNum = ReadWriteIOUtils.readInt(buffer);

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/OperateMultiplePipesPlanV2.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@ protected void serializeImpl(DataOutputStream stream) throws IOException {
7575
@Override
7676
protected void deserializeImpl(ByteBuffer buffer) throws IOException {
7777
if (ReadWriteIOUtils.readBool(buffer)) {
78-
subPlans = new ArrayList<>();
7978
int size = ReadWriteIOUtils.readInt(buffer);
79+
subPlans = new ArrayList<>(size);
8080
for (int i = 0; i < size; i++) {
8181
short type = ReadWriteIOUtils.readShort(buffer);
8282
if (type == ConfigPhysicalPlanType.CreatePipeV2.getPlanType()) {

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/subscription/topic/AlterMultipleTopicsPlan.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ protected void serializeImpl(DataOutputStream stream) throws IOException {
6565
@Override
6666
protected void deserializeImpl(ByteBuffer buffer) throws IOException {
6767
if (ReadWriteIOUtils.readBool(buffer)) {
68-
subPlans = new ArrayList<>();
6968
int size = ReadWriteIOUtils.readInt(buffer);
69+
subPlans = new ArrayList<>(size);
7070
for (int i = 0; i < size; i++) {
7171
// This will read the type of the plan (AlterTopicPlan), which will be ignored
7272
ReadWriteIOUtils.readShort(buffer);

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.apache.iotdb.pipe.api.exception.PipeException;
4949
import org.apache.iotdb.rpc.TSStatusCode;
5050

51+
import com.google.common.collect.Maps;
5152
import org.apache.tsfile.utils.ReadWriteIOUtils;
5253
import org.slf4j.Logger;
5354
import org.slf4j.LoggerFactory;
@@ -378,7 +379,7 @@ public void deserialize(ByteBuffer byteBuffer) {
378379
super.deserialize(byteBuffer);
379380
queryId = ReadWriteIOUtils.readString(byteBuffer);
380381
int size = ReadWriteIOUtils.readInt(byteBuffer);
381-
Map<PartialPath, List<Template>> templateSetInfo = new HashMap<>();
382+
Map<PartialPath, List<Template>> templateSetInfo = Maps.newHashMapWithExpectedSize(size);
382383
for (int i = 0; i < size; i++) {
383384
PartialPath pattern = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer);
384385
int templateNum = ReadWriteIOUtils.readInt(byteBuffer);

0 commit comments

Comments
 (0)