Skip to content

Preserve LOAD failure details in logs and client responses #18935

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jt2594838 merged 2 commits into from
Oct 20, 2026
Merged

Preserve LOAD failure details in logs and client responses #18935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cf8c0e2
Preserve LOAD failure details in logs and client responses
Caideyipi Oct 20, 2026
41adfef
Declare the PowerMock reflection test dependency
Caideyipi Oct 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions iotdb-core/datanode/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,11 @@
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-reflect</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
Expand Down
17 changes: 17 additions & 0 deletions iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,23 @@
package org.apache.iotdb.db.i18n;

public final class DataNodeQueryMessages {
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477 =
"Failed to convert TsFile %s to tablets. Status: %s";
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD =
"Failed to convert TsFile %s to tablets: %s";
public static final String MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3 =
"Tablet conversion returned no status";
public static final String EXCEPTION_FAILED_TO_READ_TSFILE_MODEL_INFORMATION_BEFORE_TABLET_CONVERSION_FOR_STATEMENT_ARG_ARG_F05C852A =
"Failed to read TsFile model information before tablet conversion for statement %s: %s";
public static final String EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5 =
"Failed to load TsFiles after tablet conversion. Details: %s";
public static final String LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E =
"LOAD rejected: uuid={}, region={}, endpoint={}, status={}";
public static final String MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645 =
"Target node rejected LOAD without a status or a message";
public static final String MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A =
"Failed to dispatch LOAD %s for region %s to node %s: %s";

public static final String MESSAGE_SCHEMA_REGION_IS_UNAVAILABLE_PLEASE_RETRY_LATER_D642510A =
"Schema region is unavailable. Please retry later.";

Expand Down
17 changes: 17 additions & 0 deletions iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,23 @@
package org.apache.iotdb.db.i18n;

public final class DataNodeQueryMessages {
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477 =
"将 TsFile %s 转换为 tablet 失败。状态：%s";
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD =
"将 TsFile %s 转换为 tablet 失败：%s";
public static final String MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3 =
"tablet 转换未返回状态";
public static final String EXCEPTION_FAILED_TO_READ_TSFILE_MODEL_INFORMATION_BEFORE_TABLET_CONVERSION_FOR_STATEMENT_ARG_ARG_F05C852A =
"执行语句 %s 时，在转换为 tablet 前读取 TsFile 模型信息失败：%s";
public static final String EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5 =
"将 TsFile 转换为 tablet 后加载仍失败。详情：%s";
public static final String LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E =
"LOAD 被拒绝：uuid={}，region={}，endpoint={}，status={}";
public static final String MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645 =
"目标节点拒绝 LOAD，且未返回状态或消息";
public static final String MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A =
"分发 LOAD %s（region %s）到节点 %s 失败：%s";

public static final String MESSAGE_SCHEMA_REGION_IS_UNAVAILABLE_PLEASE_RETRY_LATER_D642510A =
"SchemaRegion 不可用，请稍后重试。";

Expand Down
56 changes: 33 additions & 23 deletions ...e/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -804,13 +804,15 @@ private void executeTabletConversionOnException(
try {
getFileModelInfoBeforeTabletConversion();
} catch (Exception e1) {
LOGGER.warn(
DataNodeQueryMessages
.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_STATEMENT_ARG_BECAUSE_FAILED_TO_READ_MODEL_INFO,
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement,
e1.getMessage());
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_READ_TSFILE_MODEL_INFORMATION_BEFORE_TABLET_CONVERSION_FOR_STATEMENT_ARG_ARG_F05C852A,
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement,
e1);
LOGGER.warn(message, e1);
analysis.setFailStatus(
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage()));
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(message));
analysis.setFinishQueryAfterAnalyze(true);
setRealStatement(analysis);
return;
Expand All @@ -822,6 +824,7 @@ private void executeTabletConversionOnException(

for (int i = 0; i < tsFiles.size(); i++) {
final long startTime = System.nanoTime();
final String filePath = tsFiles.get(i).getPath();
try {
final TSStatus status =
isTableModelTsFile.get(i)
Expand All @@ -846,30 +849,37 @@ private void executeTabletConversionOnException(
.orElse(null);

if (status == null) {
LOGGER.warn(
DataNodeQueryMessages
.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_STATEMENT_ARG_STATUS_IS_NULL,
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement);
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
filePath,
DataNodeQueryMessages.MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3);
LOGGER.warn(message);
analysis.setFailStatus(
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode())
.setMessage(e.getMessage()));
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(message));
break;
} else if (!loadTsFileDataTypeConverter.isSuccessful(status)) {
LOGGER.warn(
DataNodeQueryMessages.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_STATEMENT_ARG_STATUS_ARG,
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement,
status);
analysis.setFailStatus(status);
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477,
filePath,
status);
LOGGER.warn(message);
analysis.setFailStatus(status.deepCopy().setMessage(message));
break;
}
} catch (final Exception e2) {
LOGGER.warn(
DataNodeQueryMessages
.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_STATEMENT_ARG_BECAUSE_EXCEPTION_ARG,
isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement,
e2.getMessage());
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
filePath,
e2);
LOGGER.warn(message, e2);
analysis.setFailStatus(
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage()));
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(message));
break;
} finally {
LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost(
Expand Down
56 changes: 33 additions & 23 deletions ...rc/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.slf4j.LoggerFactory;

import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
Expand Down Expand Up @@ -113,6 +114,7 @@ public void convertFailedTsFilesToTablets() {
private void convertAndRetry() {
final LoadTsFileDataTypeConverter converter =
new LoadTsFileDataTypeConverter(queryContext, isGeneratedByPipe);
final List<String> failureDetails = new ArrayList<>();

final Iterator<Integer> iterator = failedTsFileNodeIndexes.iterator();
while (iterator.hasNext()) {
Expand Down Expand Up @@ -141,21 +143,34 @@ private void convertAndRetry() {
.LOAD_SUCCESSFULLY_CONVERTED_TSFILE_ARG_INTO_TABLETS_AND_INSERTED,
filePath);
} else {
LOGGER.warn(
DataNodeQueryMessages.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_TSFILE_ARG_STATUS_ARG,
filePath,
status);
final String message =
status == null
? String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
filePath,
DataNodeQueryMessages.MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3)
: String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477,
filePath,
status);
failureDetails.add(message);
LOGGER.warn(message);
}
} catch (final Exception e) {
LOGGER.warn(
DataNodeQueryMessages.LOAD_FAILED_TO_CONVERT_TO_TABLETS_FROM_TSFILE_ARG_EXCEPTION_ARG,
filePath,
e.getMessage(),
e);
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
filePath,
e);
failureDetails.add(message);
LOGGER.warn(message, e);
}
}

resolveFinalState();
resolveFinalState(failureDetails);
}

private TSStatus executeTableModelConversion(
Expand Down Expand Up @@ -185,25 +200,20 @@ private TSStatus executeTreeModelConversion(
return converter.convertForTreeModel(statement).orElse(null);
}

private void resolveFinalState() {
private void resolveFinalState(final List<String> failureDetails) {
if (failedTsFileNodeIndexes.isEmpty()) {
LOGGER.info(DataNodeQueryMessages.LOAD_ALL_FAILED_TSFILES_ARE_CONVERTED_TO_TABLETS);
stateMachine.transitionToFinished();
return;
}

final String remainingFailedFiles = getFailedFilePathsString();
LOGGER.warn(
DataNodeQueryMessages
.LOG_LOAD_FAILED_TO_LOAD_SOME_TSFILES_BY_CONVERTING_THEM_INTO_TABLETS_FAILED_TSFILES_ARG_7D9DB9C3,
remainingFailedFiles);

stateMachine.transitionToFailed(
new LoadFileException(
String.format(
DataNodeQueryMessages
.LOG_LOAD_FAILED_TO_LOAD_SOME_TSFILES_BY_CONVERTING_THEM_INTO_TABLETS_FAILED_TSFILES_ARG_7D9DB9C3,
remainingFailedFiles)));
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5,
String.join("; ", failureDetails));
LOGGER.warn(message);
stateMachine.transitionToFailed(new LoadFileException(message));
}

// -------------------------------------------------------------------------
Expand Down
25 changes: 22 additions & 3 deletions ...in/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,19 +260,37 @@ private void dispatchRemote(
for (final TTsFilePieceReq request : requests) {
final TLoadResp loadResp = client.sendTsFilePieceNode(request);
if (!loadResp.isAccepted()) {
final TSStatus failureStatus =
loadResp.isSetStatus()
? loadResp.getStatus()
: RpcUtils.getStatus(
TSStatusCode.DISPATCH_ERROR,
loadResp.isSetMessage()
? loadResp.getMessage()
: DataNodeQueryMessages
.MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645);
final String errorCode =
loadResp.isSetStatus()
? String.valueOf(loadResp.getStatus().getCode())
: UserDataTransferErrorCode.REMOTE_REJECTED.name();
recordTransferAttempt(endPoint, false, errorCode, null);
transferAttemptAudited = true;
LOGGER.warn(loadResp.getMessage());
throw new FragmentInstanceDispatchException(loadResp.getStatus());
LOGGER.warn(
DataNodeQueryMessages
.LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E,
uuid,
consensusGroupId,
endPoint,
failureStatus);
throw new FragmentInstanceDispatchException(failureStatus);
}
}
recordTransferAttempt(endPoint, true, null, null);
transferAttemptAudited = true;
}
} catch (final FragmentInstanceDispatchException e) {
// A receiver rejection is already audited and carries its original status and retry hints.
throw e;
} catch (final Exception e) {
if (!transferAttemptAudited) {
recordTransferAttempt(endPoint, false, null, e);
Expand All @@ -282,8 +300,9 @@ private void dispatchRemote(
final String message =
String.format(
DataNodeQueryMessages
.MESSAGE_FAILED_TO_DISPATCH_LOAD_COMMAND_ARG_TO_NODE_ARG_BECAUSE_OF_EXCEPTION_ARG_2D8A483D,
.MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A,
uuid,
consensusGroupId,
endPoint,
e);
LOGGER.warn(message, e);
Expand Down
Loading
Loading