4343import java .util .ArrayList ;
4444import java .util .List ;
4545import java .util .Optional ;
46+ import java .util .function .Function ;
4647import java .util .stream .Collectors ;
4748
4849import static org .apache .iotdb .db .pipe .resource .memory .PipeMemoryWeightUtil .calculateTabletSizeInBytes ;
@@ -58,6 +59,7 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
5859 .getLoadTsFileTabletConversionBatchMemorySizeInBytes ();
5960
6061 private final StatementExecutor statementExecutor ;
62+ private final Function <File , LoadTreeTsFileTabletIterator > tabletIteratorFactory ;
6163
6264 @ FunctionalInterface
6365 public interface StatementExecutor {
@@ -66,7 +68,14 @@ public interface StatementExecutor {
6668
6769 public LoadTreeStatementDataTypeConvertExecutionVisitor (
6870 final StatementExecutor statementExecutor ) {
71+ this (statementExecutor , file -> new LoadTreeTsFileTabletIterator (file , true ));
72+ }
73+
74+ LoadTreeStatementDataTypeConvertExecutionVisitor (
75+ final StatementExecutor statementExecutor ,
76+ final Function <File , LoadTreeTsFileTabletIterator > tabletIteratorFactory ) {
6977 this .statementExecutor = statementExecutor ;
78+ this .tabletIteratorFactory = tabletIteratorFactory ;
7079 }
7180
7281 @ Override
@@ -89,7 +98,7 @@ public Optional<TSStatus> visitLoadFile(
8998 try {
9099 for (final File file : loadTsFileStatement .getTsFiles ()) {
91100 try (final LoadTreeTsFileTabletIterator tabletIterator =
92- new LoadTreeTsFileTabletIterator (file , true )) {
101+ tabletIteratorFactory . apply (file )) {
93102 for (final Pair <Tablet , Boolean > tabletWithIsAligned : tabletIterator ) {
94103 final PipeTransferTabletRawReq tabletRawReq =
95104 PipeTransferTabletRawReq .toTPipeTransferRawReq (
@@ -123,9 +132,29 @@ public Optional<TSStatus> visitLoadFile(
123132 } catch (final Exception e ) {
124133 LOGGER .warn (
125134 "Failed to convert data type for LoadTsFileStatement: {}." , loadTsFileStatement , e );
126- return Optional . of (
135+ final TSStatus status =
127136 loadTsFileStatement .accept (
128- LoadTsFileDataTypeConverter .STATEMENT_EXCEPTION_VISITOR , e ));
137+ LoadTsFileDataTypeConverter .STATEMENT_EXCEPTION_VISITOR , e );
138+
139+ // A parser can fail after producing tablets that are still waiting for the next batch
140+ // boundary. Submit those tablets before reporting the parser error so the error does not
141+ // discard successfully converted data.
142+ if (!isRetryableConversionException (e ) && !tabletRawReqs .isEmpty ()) {
143+ final TSStatus flushStatus =
144+ executeInsertMultiTabletsWithRetry (
145+ tabletRawReqs , loadTsFileStatement .isConvertOnTypeMismatch ());
146+
147+ for (final long memoryCost : tabletRawReqSizes ) {
148+ block .reduceMemoryUsage (memoryCost );
149+ }
150+ tabletRawReqs .clear ();
151+ tabletRawReqSizes .clear ();
152+
153+ if (!handleTSStatus (flushStatus , loadTsFileStatement )) {
154+ return Optional .of (flushStatus );
155+ }
156+ }
157+ return Optional .of (status );
129158 }
130159 }
131160
@@ -179,6 +208,21 @@ public Optional<TSStatus> visitLoadFile(
179208 return Optional .of (new TSStatus (TSStatusCode .SUCCESS_STATUS .getStatusCode ()));
180209 }
181210
211+ private static boolean isRetryableConversionException (final Throwable throwable ) {
212+ if (LoadTsFileDataTypeConverter .isMemoryPressureException (throwable )) {
213+ return true ;
214+ }
215+
216+ Throwable current = throwable ;
217+ while (current != null ) {
218+ if (current instanceof InterruptedException ) {
219+ return true ;
220+ }
221+ current = current .getCause ();
222+ }
223+ return false ;
224+ }
225+
182226 private TSStatus executeInsertMultiTabletsWithRetry (
183227 final List <PipeTransferTabletRawReq > tabletRawReqs , boolean isConvertOnTypeMismatch ) {
184228 final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement ();
0 commit comments