Skip to content

Commit 9904ca3

Browse files
CaideyipiCaideyipi
authored
Preserve LOAD failure details in logs and client responses (#18935)
* Preserve LOAD failure details in logs and client responses * Declare the PowerMock reflection test dependency
1 parent c75e9a8 commit 9904ca3

9 files changed

Lines changed: 566 additions & 54 deletions

File tree

‎iotdb-core/datanode/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,11 @@
289289
<artifactId>powermock-api-mockito2</artifactId>
290290
<scope>test</scope>
291291
</dependency>
292+
<dependency>
293+
<groupId>org.powermock</groupId>
294+
<artifactId>powermock-reflect</artifactId>
295+
<scope>test</scope>
296+
</dependency>
292297
<dependency>
293298
<groupId>org.awaitility</groupId>
294299
<artifactId>awaitility</artifactId>

‎iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,23 @@
2020
package org.apache.iotdb.db.i18n;
2121

2222
public final class DataNodeQueryMessages {
23+
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477 =
24+
"Failed to convert TsFile %s to tablets. Status: %s";
25+
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD =
26+
"Failed to convert TsFile %s to tablets: %s";
27+
public static final String MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3 =
28+
"Tablet conversion returned no status";
29+
public static final String EXCEPTION_FAILED_TO_READ_TSFILE_MODEL_INFORMATION_BEFORE_TABLET_CONVERSION_FOR_STATEMENT_ARG_ARG_F05C852A =
30+
"Failed to read TsFile model information before tablet conversion for statement %s: %s";
31+
public static final String EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5 =
32+
"Failed to load TsFiles after tablet conversion. Details: %s";
33+
public static final String LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E =
34+
"LOAD rejected: uuid={}, region={}, endpoint={}, status={}";
35+
public static final String MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645 =
36+
"Target node rejected LOAD without a status or a message";
37+
public static final String MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A =
38+
"Failed to dispatch LOAD %s for region %s to node %s: %s";
39+
2340
public static final String MESSAGE_SCHEMA_REGION_IS_UNAVAILABLE_PLEASE_RETRY_LATER_D642510A =
2441
"Schema region is unavailable. Please retry later.";
2542

‎iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,23 @@
2020
package org.apache.iotdb.db.i18n;
2121

2222
public final class DataNodeQueryMessages {
23+
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477 =
24+
"将 TsFile %s 转换为 tablet 失败。状态：%s";
25+
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD =
26+
"将 TsFile %s 转换为 tablet 失败：%s";
27+
public static final String MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3 =
28+
"tablet 转换未返回状态";
29+
public static final String EXCEPTION_FAILED_TO_READ_TSFILE_MODEL_INFORMATION_BEFORE_TABLET_CONVERSION_FOR_STATEMENT_ARG_ARG_F05C852A =
30+
"执行语句 %s 时，在转换为 tablet 前读取 TsFile 模型信息失败：%s";
31+
public static final String EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5 =
32+
"将 TsFile 转换为 tablet 后加载仍失败。详情：%s";
33+
public static final String LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E =
34+
"LOAD 被拒绝：uuid={}，region={}，endpoint={}，status={}";
35+
public static final String MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645 =
36+
"目标节点拒绝 LOAD，且未返回状态或消息";
37+
public static final String MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A =
38+
"分发 LOAD %s（region %s）到节点 %s 失败：%s";
39+
2340
public static final String MESSAGE_SCHEMA_REGION_IS_UNAVAILABLE_PLEASE_RETRY_LATER_D642510A =
2441
"SchemaRegion 不可用，请稍后重试。";
2542

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java‎

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -804,13 +804,15 @@ private void executeTabletConversionOnException(
804804
try {
805805
getFileModelInfoBeforeTabletConversion();
806806
} catch (Exception e1) {
807-
LOGGER.warn(
808-
DataNodeQueryMessages
809-
.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_STATEMENT_ARG_BECAUSE_FAILED_TO_READ_MODEL_INFO,
810-
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement,
811-
e1.getMessage());
807+
final String message =
808+
String.format(
809+
DataNodeQueryMessages
810+
.EXCEPTION_FAILED_TO_READ_TSFILE_MODEL_INFORMATION_BEFORE_TABLET_CONVERSION_FOR_STATEMENT_ARG_ARG_F05C852A,
811+
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement,
812+
e1);
813+
LOGGER.warn(message, e1);
812814
analysis.setFailStatus(
813-
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage()));
815+
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(message));
814816
analysis.setFinishQueryAfterAnalyze(true);
815817
setRealStatement(analysis);
816818
return;
@@ -822,6 +824,7 @@ private void executeTabletConversionOnException(
822824

823825
for (int i = 0; i < tsFiles.size(); i++) {
824826
final long startTime = System.nanoTime();
827+
final String filePath = tsFiles.get(i).getPath();
825828
try {
826829
final TSStatus status =
827830
isTableModelTsFile.get(i)
@@ -846,30 +849,37 @@ private void executeTabletConversionOnException(
846849
.orElse(null);
847850

848851
if (status == null) {
849-
LOGGER.warn(
850-
DataNodeQueryMessages
851-
.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_STATEMENT_ARG_STATUS_IS_NULL,
852-
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement);
852+
final String message =
853+
String.format(
854+
DataNodeQueryMessages
855+
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
856+
filePath,
857+
DataNodeQueryMessages.MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3);
858+
LOGGER.warn(message);
853859
analysis.setFailStatus(
854-
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode())
855-
.setMessage(e.getMessage()));
860+
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(message));
856861
break;
857862
} else if (!loadTsFileDataTypeConverter.isSuccessful(status)) {
858-
LOGGER.warn(
859-
DataNodeQueryMessages.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_STATEMENT_ARG_STATUS_ARG,
860-
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement,
861-
status);
862-
analysis.setFailStatus(status);
863+
final String message =
864+
String.format(
865+
DataNodeQueryMessages
866+
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477,
867+
filePath,
868+
status);
869+
LOGGER.warn(message);
870+
analysis.setFailStatus(status.deepCopy().setMessage(message));
863871
break;
864872
}
865873
} catch (final Exception e2) {
866-
LOGGER.warn(
867-
DataNodeQueryMessages
868-
.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_STATEMENT_ARG_BECAUSE_EXCEPTION_ARG,
869-
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement,
870-
e2.getMessage());
874+
final String message =
875+
String.format(
876+
DataNodeQueryMessages
877+
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
878+
filePath,
879+
e2);
880+
LOGGER.warn(message, e2);
871881
analysis.setFailStatus(
872-
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage()));
882+
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(message));
873883
break;
874884
} finally {
875885
LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandler.java‎

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.slf4j.LoggerFactory;
3535

3636
import java.io.FileNotFoundException;
37+
import java.util.ArrayList;
3738
import java.util.Collections;
3839
import java.util.Iterator;
3940
import java.util.List;
@@ -113,6 +114,7 @@ public void convertFailedTsFilesToTablets() {
113114
private void convertAndRetry() {
114115
final LoadTsFileDataTypeConverter converter =
115116
new LoadTsFileDataTypeConverter(queryContext, isGeneratedByPipe);
117+
final List<String> failureDetails = new ArrayList<>();
116118

117119
final Iterator<Integer> iterator = failedTsFileNodeIndexes.iterator();
118120
while (iterator.hasNext()) {
@@ -141,21 +143,34 @@ private void convertAndRetry() {
141143
.LOAD_SUCCESSFULLY_CONVERTED_TSFILE_ARG_INTO_TABLETS_AND_INSERTED,
142144
filePath);
143145
} else {
144-
LOGGER.warn(
145-
DataNodeQueryMessages.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_TSFILE_ARG_STATUS_ARG,
146-
filePath,
147-
status);
146+
final String message =
147+
status == null
148+
? String.format(
149+
DataNodeQueryMessages
150+
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
151+
filePath,
152+
DataNodeQueryMessages.MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3)
153+
: String.format(
154+
DataNodeQueryMessages
155+
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477,
156+
filePath,
157+
status);
158+
failureDetails.add(message);
159+
LOGGER.warn(message);
148160
}
149161
} catch (final Exception e) {
150-
LOGGER.warn(
151-
DataNodeQueryMessages.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_TSFILE_ARG_EXCEPTION_ARG,
152-
filePath,
153-
e.getMessage(),
154-
e);
162+
final String message =
163+
String.format(
164+
DataNodeQueryMessages
165+
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
166+
filePath,
167+
e);
168+
failureDetails.add(message);
169+
LOGGER.warn(message, e);
155170
}
156171
}
157172

158-
resolveFinalState();
173+
resolveFinalState(failureDetails);
159174
}
160175

161176
private TSStatus executeTableModelConversion(
@@ -185,25 +200,20 @@ private TSStatus executeTreeModelConversion(
185200
return converter.convertForTreeModel(statement).orElse(null);
186201
}
187202

188-
private void resolveFinalState() {
203+
private void resolveFinalState(final List<String> failureDetails) {
189204
if (failedTsFileNodeIndexes.isEmpty()) {
190205
LOGGER.info(DataNodeQueryMessages.LOAD_ALL_FAILED_TSFILES_ARE_CONVERTED_TO_TABLETS);
191206
stateMachine.transitionToFinished();
192207
return;
193208
}
194209

195-
final String remainingFailedFiles = getFailedFilePathsString();
196-
LOGGER.warn(
197-
DataNodeQueryMessages
198-
.LOG_LOAD_FAILED_TO_LOAD_SOME_TSFILES_BY_CONVERTING_THEM_INTO_TABLETS_FAILED_TSFILES_ARG_7D9DB9C3,
199-
remainingFailedFiles);
200-
201-
stateMachine.transitionToFailed(
202-
new LoadFileException(
203-
String.format(
204-
DataNodeQueryMessages
205-
.LOG_LOAD_FAILED_TO_LOAD_SOME_TSFILES_BY_CONVERTING_THEM_INTO_TABLETS_FAILED_TSFILES_ARG_7D9DB9C3,
206-
remainingFailedFiles)));
210+
final String message =
211+
String.format(
212+
DataNodeQueryMessages
213+
.EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5,
214+
String.join("; ", failureDetails));
215+
LOGGER.warn(message);
216+
stateMachine.transitionToFailed(new LoadFileException(message));
207217
}
208218

209219
// -------------------------------------------------------------------------

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -260,19 +260,37 @@ private void dispatchRemote(
260260
for (final TTsFilePieceReq request : requests) {
261261
final TLoadResp loadResp = client.sendTsFilePieceNode(request);
262262
if (!loadResp.isAccepted()) {
263+
final TSStatus failureStatus =
264+
loadResp.isSetStatus()
265+
? loadResp.getStatus()
266+
: RpcUtils.getStatus(
267+
TSStatusCode.DISPATCH_ERROR,
268+
loadResp.isSetMessage()
269+
? loadResp.getMessage()
270+
: DataNodeQueryMessages
271+
.MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645);
263272
final String errorCode =
264273
loadResp.isSetStatus()
265274
? String.valueOf(loadResp.getStatus().getCode())
266275
: UserDataTransferErrorCode.REMOTE_REJECTED.name();
267276
recordTransferAttempt(endPoint, false, errorCode, null);
268277
transferAttemptAudited = true;
269-
LOGGER.warn(loadResp.getMessage());
270-
throw new FragmentInstanceDispatchException(loadResp.getStatus());
278+
LOGGER.warn(
279+
DataNodeQueryMessages
280+
.LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E,
281+
uuid,
282+
consensusGroupId,
283+
endPoint,
284+
failureStatus);
285+
throw new FragmentInstanceDispatchException(failureStatus);
271286
}
272287
}
273288
recordTransferAttempt(endPoint, true, null, null);
274289
transferAttemptAudited = true;
275290
}
291+
} catch (final FragmentInstanceDispatchException e) {
292+
// A receiver rejection is already audited and carries its original status and retry hints.
293+
throw e;
276294
} catch (final Exception e) {
277295
if (!transferAttemptAudited) {
278296
recordTransferAttempt(endPoint, false, null, e);
@@ -282,8 +300,9 @@ private void dispatchRemote(
282300
final String message =
283301
String.format(
284302
DataNodeQueryMessages
285-
.MESSAGE_FAILED_TO_DISPATCH_LOAD_COMMAND_ARG_TO_NODE_ARG_BECAUSE_OF_EXCEPTION_ARG_2D8A483D,
303+
.MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A,
286304
uuid,
305+
consensusGroupId,
287306
endPoint,
288307
e);
289308
LOGGER.warn(message, e);

0 commit comments

Comments
 (0)