Skip to content

Commit 8e471a2

Browse files
jr-rkclaude
andcommitted
fix(mediafilter): also skip unparsable docs on the Tika use-temp-file path
The textextractor.use-temp-file branch streamed the source straight into parser.parse(), so a corrupt PDF there still threw and produced the same filter-media ERROR flood the in-memory path now avoids. Buffer the bitstream to a temp input file first (a genuine assetstore read failure surfaces from the copy and propagates), then parse the local copy and catch IOException/ TikaException as an unparsable-document failure -> WARN + skip. Keeps text extraction off-heap for large files, matching the in-memory read/parse split. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent faf6d8a commit 8e471a2

1 file changed

Lines changed: 28 additions & 6 deletions

File tree

dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import java.io.IOException;
1515
import java.io.InputStream;
1616
import java.nio.charset.StandardCharsets;
17+
import java.nio.file.Files;
18+
import java.nio.file.StandardCopyOption;
1719

1820
import org.apache.commons.lang.StringUtils;
1921
import org.apache.logging.log4j.LogManager;
@@ -69,7 +71,7 @@ public InputStream getDestinationStream(Item currentItem, InputStream source, bo
6971

7072
if (useTemporaryFile) {
7173
// Extract text out of source file using a temp file, returning results as InputStream
72-
return extractUsingTempFile(source, verbose);
74+
return extractUsingTempFile(currentItem, source, verbose);
7375
}
7476

7577
// Not using temporary file. We'll use Tika's default in-memory parsing.
@@ -139,15 +141,25 @@ private void handleOutOfMemory(Item currentItem, OutOfMemoryError oe) {
139141
/**
140142
* Extracts the text out of a given source InputStream, using a temporary file. This decreases the amount of memory
141143
* necessary for text extraction, but can be slower as it requires writing extracted text to a temporary file.
144+
* @param currentItem the Item whose bitstream is being processed (used for logging)
142145
* @param source source InputStream
143146
* @param verbose verbose mode enabled/disabled
144-
* @return InputStream for temporary file containing extracted text
147+
* @return InputStream for temporary file containing extracted text, or null if the source could not be parsed
145148
* @throws IOException
146149
* @throws SAXException
147150
* @throws TikaException
148151
*/
149-
private InputStream extractUsingTempFile(InputStream source, boolean verbose)
152+
private InputStream extractUsingTempFile(Item currentItem, InputStream source, boolean verbose)
150153
throws IOException, TikaException, SAXException {
154+
// Buffer the bitstream to a temporary input file first. A genuine assetstore read failure
155+
// (disk/network/S3) surfaces from this copy and propagates; the parse below then reads a local
156+
// file, so a parse-time failure can only be an unparsable document. This keeps extraction
157+
// off-heap for large files (the reason this path exists) while still distinguishing read
158+
// failures from data-quality failures, exactly like the in-memory path.
159+
File tempSourceFile = File.createTempFile("dspacetextsource" + source.hashCode(), ".bin");
160+
tempSourceFile.deleteOnExit();
161+
Files.copy(source, tempSourceFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
162+
151163
File tempExtractedTextFile = File.createTempFile("dspacetextextract" + source.hashCode(), ".txt");
152164

153165
if (verbose) {
@@ -158,7 +170,8 @@ private InputStream extractUsingTempFile(InputStream source, boolean verbose)
158170
}
159171

160172
// Open temp file for writing
161-
try (FileWriter writer = new FileWriter(tempExtractedTextFile, StandardCharsets.UTF_8)) {
173+
try (FileWriter writer = new FileWriter(tempExtractedTextFile, StandardCharsets.UTF_8);
174+
InputStream bufferedSource = new FileInputStream(tempSourceFile)) {
162175
// Initialize a custom ContentHandlerDecorator which is a BodyContentHandler.
163176
// This mimics the behavior of Tika().parseToString(), which only extracts text from the body of the file.
164177
// This custom Handler writes any extracted text to the temp file.
@@ -204,8 +217,17 @@ public void ignorableWhitespace(char[] ch, int start, int length) throws SAXExce
204217

205218
AutoDetectParser parser = new AutoDetectParser();
206219
Metadata metadata = new Metadata();
207-
// parse our source InputStream using the above custom handler
208-
parser.parse(source, handler, metadata);
220+
// parse the buffered copy using the above custom handler
221+
parser.parse(bufferedSource, handler, metadata);
222+
} catch (IOException | TikaException e) {
223+
// Parsing the already-buffered copy failed -> malformed/non-standard document (corrupt PDF
224+
// header, missing xref, truncated content), not an assetstore read error. Skip instead of
225+
// failing the whole filter-media run. See dataquest-dev/dspace-customers#903.
226+
log.warn("Unable to extract text from bitstream in Item {}: {}",
227+
currentItem.getHandle(), e.getMessage());
228+
return null;
229+
} finally {
230+
Files.deleteIfExists(tempSourceFile.toPath());
209231
}
210232

211233
// At this point, all extracted text is written to our temp file. So, return a FileInputStream for that file

0 commit comments

Comments
 (0)