Skip to content

Commit ba30d3f

Browse files
andrestubbeandrestubbe
committed
Auto Update
1 parent 3f9c131 commit ba30d3f

3 files changed

Lines changed: 245 additions & 254 deletions

File tree

‎src/main/java/fastcontentparse/FastContentParse.java‎

Lines changed: 78 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.util.ArrayList;
99
import java.util.List;
1010
import java.util.Locale;
11+
import java.util.Map;
1112
import java.util.regex.Pattern;
1213

1314
import org.apache.pdfbox.Loader;
@@ -127,35 +128,42 @@ private String normalizeWhitespace(String text) {
127128
return fastregex.FastRegex.normalizeWhitespace(text);
128129
}
129130

131+
private static final javax.xml.stream.XMLInputFactory XML_FACTORY;
132+
private static final Map<String, String> EXTENSION_TYPES = Map.of(
133+
".pdf", "application/pdf",
134+
".rtf", "text/rtf",
135+
".md", "text/markdown",
136+
".markdown", "text/markdown",
137+
".doc", "application/msword",
138+
".docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
139+
".csv", "text/csv",
140+
".xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
141+
".png", "image/ocr",
142+
".jpg", "image/ocr"
143+
);
144+
145+
static {
146+
XML_FACTORY = javax.xml.stream.XMLInputFactory.newDefaultFactory();
147+
XML_FACTORY.setProperty(javax.xml.stream.XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
148+
XML_FACTORY.setProperty(javax.xml.stream.XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
149+
}
150+
130151
private String detectType(String sourceName) {
131152
if (sourceName == null) {
132153
return "text/plain";
133154
}
134155

135156
String lower = sourceName.toLowerCase(Locale.ROOT);
136-
if (lower.endsWith(".pdf")) {
137-
return "application/pdf";
138-
}
139-
if (lower.endsWith(".rtf")) {
140-
return "text/rtf";
141-
}
142-
if (lower.endsWith(".md") || lower.endsWith(".markdown")) {
143-
return "text/markdown";
144-
}
145-
if (lower.endsWith(".doc")) {
146-
return "application/msword";
147-
}
148-
if (lower.endsWith(".docx")) {
149-
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
150-
}
151-
if (lower.endsWith(".csv")) {
152-
return "text/csv";
153-
}
154-
if (lower.endsWith(".xlsx")) {
155-
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
156-
}
157-
if (lower.endsWith(".png") || lower.endsWith(".jpg") || lower.endsWith(".jpeg") || lower.endsWith(".bmp")) {
158-
return "image/ocr";
157+
int dot = lower.lastIndexOf('.');
158+
if (dot != -1) {
159+
String ext = lower.substring(dot);
160+
String mapped = EXTENSION_TYPES.get(ext);
161+
if (mapped != null) {
162+
return mapped;
163+
}
164+
if (".jpeg".equals(ext) || ".bmp".equals(ext)) {
165+
return "image/ocr";
166+
}
159167
}
160168
return "text/plain";
161169
}
@@ -168,17 +176,14 @@ private ParsedDocument parseCsv(Path path) throws IOException {
168176

169177
private ParsedDocument parseXlsx(Path path) throws IOException {
170178
List<String> sharedStrings = new ArrayList<>();
171-
StringBuilder textBuilder = new StringBuilder();
179+
StringBuilder textBuilder = new StringBuilder(128 * 1024);
172180

173181
try (java.util.zip.ZipFile zip = new java.util.zip.ZipFile(path.toFile())) {
174182
// 1. Read sharedStrings.xml if present
175183
java.util.zip.ZipEntry sstEntry = zip.getEntry("xl/sharedStrings.xml");
176184
if (sstEntry != null) {
177185
try (InputStream is = zip.getInputStream(sstEntry)) {
178-
javax.xml.stream.XMLInputFactory factory = javax.xml.stream.XMLInputFactory.newDefaultFactory();
179-
factory.setProperty(javax.xml.stream.XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
180-
factory.setProperty(javax.xml.stream.XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
181-
javax.xml.stream.XMLStreamReader reader = factory.createXMLStreamReader(is);
186+
javax.xml.stream.XMLStreamReader reader = XML_FACTORY.createXMLStreamReader(is);
182187

183188
StringBuilder currentText = null;
184189
while (reader.hasNext()) {
@@ -205,55 +210,56 @@ private ParsedDocument parseXlsx(Path path) throws IOException {
205210
}
206211
}
207212

208-
// 2. Read sheet1.xml (primary sheet)
209-
java.util.zip.ZipEntry sheetEntry = zip.getEntry("xl/worksheets/sheet1.xml");
210-
if (sheetEntry != null) {
211-
try (InputStream is = zip.getInputStream(sheetEntry)) {
212-
javax.xml.stream.XMLInputFactory factory = javax.xml.stream.XMLInputFactory.newDefaultFactory();
213-
factory.setProperty(javax.xml.stream.XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
214-
factory.setProperty(javax.xml.stream.XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
215-
javax.xml.stream.XMLStreamReader reader = factory.createXMLStreamReader(is);
216-
217-
String cellType = null;
218-
StringBuilder cellVal = null;
219-
220-
while (reader.hasNext()) {
221-
int event = reader.next();
222-
if (event == javax.xml.stream.XMLStreamConstants.START_ELEMENT) {
223-
String name = reader.getLocalName();
224-
if ("c".equals(name)) {
225-
cellType = reader.getAttributeValue(null, "t");
226-
} else if ("v".equals(name)) {
227-
cellVal = new StringBuilder();
228-
}
229-
} else if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS) {
230-
if (cellVal != null) {
231-
cellVal.append(reader.getText());
232-
}
233-
} else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) {
234-
String name = reader.getLocalName();
235-
if ("v".equals(name)) {
213+
// 2. Read all sheets (sheet1.xml, sheet2.xml, etc.)
214+
java.util.Enumeration<? extends java.util.zip.ZipEntry> entries = zip.entries();
215+
while (entries.hasMoreElements()) {
216+
java.util.zip.ZipEntry entry = entries.nextElement();
217+
String entryName = entry.getName();
218+
if (entryName.startsWith("xl/worksheets/sheet") && entryName.endsWith(".xml")) {
219+
try (InputStream is = zip.getInputStream(entry)) {
220+
javax.xml.stream.XMLStreamReader reader = XML_FACTORY.createXMLStreamReader(is);
221+
222+
String cellType = null;
223+
StringBuilder cellVal = null;
224+
225+
while (reader.hasNext()) {
226+
int event = reader.next();
227+
if (event == javax.xml.stream.XMLStreamConstants.START_ELEMENT) {
228+
String name = reader.getLocalName();
229+
if ("c".equals(name)) {
230+
cellType = reader.getAttributeValue(null, "t");
231+
} else if ("v".equals(name)) {
232+
cellVal = new StringBuilder();
233+
}
234+
} else if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS) {
236235
if (cellVal != null) {
237-
String rawVal = cellVal.toString().trim();
238-
if ("s".equals(cellType)) {
239-
try {
240-
int idx = Integer.parseInt(rawVal);
241-
if (idx >= 0 && idx < sharedStrings.size()) {
242-
textBuilder.append(sharedStrings.get(idx)).append("\t");
243-
}
244-
} catch (NumberFormatException ignored) {}
245-
} else if (!rawVal.isEmpty()) {
246-
textBuilder.append(rawVal).append("\t");
236+
cellVal.append(reader.getText());
237+
}
238+
} else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) {
239+
String name = reader.getLocalName();
240+
if ("v".equals(name)) {
241+
if (cellVal != null) {
242+
String rawVal = cellVal.toString().trim();
243+
if ("s".equals(cellType)) {
244+
try {
245+
int idx = Integer.parseInt(rawVal);
246+
if (idx >= 0 && idx < sharedStrings.size()) {
247+
textBuilder.append(sharedStrings.get(idx)).append("\t");
248+
}
249+
} catch (NumberFormatException ignored) {}
250+
} else if (!rawVal.isEmpty()) {
251+
textBuilder.append(rawVal).append("\t");
252+
}
253+
cellVal = null;
247254
}
248-
cellVal = null;
255+
} else if ("row".equals(name)) {
256+
textBuilder.append("\n");
249257
}
250-
} else if ("row".equals(name)) {
251-
textBuilder.append("\n");
252258
}
253259
}
260+
} catch (Exception e) {
261+
throw new IOException("Failed parsing XLSX worksheet " + entryName + ": " + e.getMessage(), e);
254262
}
255-
} catch (Exception e) {
256-
throw new IOException("Failed parsing XLSX worksheet: " + e.getMessage(), e);
257263
}
258264
}
259265
}
@@ -275,7 +281,7 @@ private ParsedDocument parseImageOcr(Path path) throws IOException {
275281

276282
private ParsedDocument parsePdf(Path path) throws IOException {
277283
try (PDDocument document = Loader.loadPDF(path.toFile())) {
278-
VisualParagraphPDFTextStripper2 stripper = new VisualParagraphPDFTextStripper2();
284+
VisualParagraphPDFTextStripper stripper = new VisualParagraphPDFTextStripper();
279285
stripper.getText(document); // process document text positions
280286
String raw = stripper.buildVisualText();
281287
String normalized = normalize(raw, "application/pdf");

0 commit comments

Comments
 (0)