refactor: extract XlsxStreamingParser and RtfStripper with full Javadoc

Author: andrestubbe

src/main/java/fastcontentparse/FastContentParse.java

@@ -1,22 +1,35 @@
 package fastcontentparse;
 
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.Writer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.regex.Pattern;
 
+import fastocr.FastOCR;
+import fastregex.FastRegex;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.text.PDFTextStripper;
 
+/**
+ * High-performance, zero-bloat content extraction and normalization engine.
+ * <p>
+ * Routes supported document formats (PDF, XLSX, CSV, RTF, Markdown, Plaintext, OCR images)
+ * to dedicated lightweight streaming parsers and normalizes textual output into clean UTF-8 tokens.
+ */
 public class FastContentParse {
 
+    /**
+     * Parses a file from the local file system into a {@link ParsedDocument}.
+     *
+     * @param path target file path
+     * @return normalized document representation with detected MIME type
+     * @throws NullPointerException if {@code path} is {@code null}
+     * @throws IOException if the file does not exist, cannot be read, or parsing fails
+     */
     public ParsedDocument parseFile(Path path) throws IOException {
         if (path == null) {
             throw new NullPointerException("path must not be null");
@@ -52,16 +65,36 @@ public ParsedDocument parseFile(Path path) throws IOException {
         return parseString(raw, fileName, type);
     }
 
+    /**
+     * Parses an in-memory string inferred by a source file name.
+     *
+     * @param rawText    raw text content
+     * @param sourceName original filename used for MIME type detection
+     * @return normalized document representation
+     */
     public ParsedDocument parseString(String rawText, String sourceName) {
         return parseString(rawText, sourceName, detectType(sourceName));
     }
 
+    /**
+     * Parses an in-memory string with explicit MIME type routing and normalization.
+     *
+     * @param rawText      raw text content
+     * @param sourceName   original filename or identifier
+     * @param explicitType explicit MIME type or {@code null} to infer from {@code sourceName}
+     * @return normalized document representation
+     */
     public ParsedDocument parseString(String rawText, String sourceName, String explicitType) {
         String type = explicitType != null ? explicitType : detectType(sourceName);
         String normalized = normalize(rawText, type);
         return new ParsedDocument(type, normalized);
     }
 
+    /**
+     * Deprecated method stub retained for migration compatibility.
+     *
+     * @deprecated Use the dedicated FastContentChunk library for token and character chunking.
+     */
     @Deprecated(forRemoval = true)
     public List<String> chunkText(String text, int maxChunkSize, int overlap) {
         if (text == null || text.isBlank()) {
@@ -78,7 +111,7 @@ private String normalize(String rawText, String explicitType) {
         String text = rawText.replace("\r\n", "\n").replace('\r', '\n');
 
         if (explicitType != null && explicitType.toLowerCase(Locale.ROOT).contains("rtf")) {
-            text = stripRtf(text);
+            text = RtfStripper.strip(text);
         }
 
         if (explicitType != null && explicitType.toLowerCase(Locale.ROOT).contains("pdf")) {
@@ -88,50 +121,10 @@ private String normalize(String rawText, String explicitType) {
         return normalizeWhitespace(text);
     }
 
-    private String stripRtf(String input) {
-        if (input == null || input.isEmpty()) {
-            return "";
-        }
-
-        StringBuilder sb = new StringBuilder(input.length());
-        int i = 0;
-        final int len = input.length();
-
-        while (i < len) {
-            char c = input.charAt(i);
-
-            if (c == '{' || c == '}') {
-                i++;
-                continue;
-            }
-
-            if (c == '\\') {
-                i++;
-                if (i >= len) break;
-
-                char next = input.charAt(i);
-                if (Character.isLetter(next)) {
-                    while (i < len && Character.isLetter(input.charAt(i))) i++;
-                    while (i < len && (Character.isDigit(input.charAt(i)) || input.charAt(i) == '-')) i++;
-                    if (i < len && input.charAt(i) == ' ') i++;
-                } else {
-                    i++;
-                }
-                continue;
-            }
-
-            sb.append(c);
-            i++;
-        }
-
-        return normalizeWhitespace(sb.toString());
-    }
-
     private String normalizeWhitespace(String text) {
-        return fastregex.FastRegex.normalizeWhitespace(text);
+        return FastRegex.normalizeWhitespace(text);
     }
 
-    private static final javax.xml.stream.XMLInputFactory XML_FACTORY;
     private static final Map<String, String> EXTENSION_TYPES = Map.ofEntries(
             Map.entry(".pdf", "application/pdf"),
             Map.entry(".rtf", "text/rtf"),
@@ -147,12 +140,6 @@ private String normalizeWhitespace(String text) {
             Map.entry(".bmp", "image/ocr")
     );
 
-    static {
-        XML_FACTORY = javax.xml.stream.XMLInputFactory.newDefaultFactory();
-        XML_FACTORY.setProperty(javax.xml.stream.XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
-        XML_FACTORY.setProperty(javax.xml.stream.XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
-    }
-
     private String detectType(String sourceName) {
         if (sourceName == null) {
             return "text/plain";
@@ -177,102 +164,14 @@ private ParsedDocument parseCsv(Path path) throws IOException {
     }
 
     private ParsedDocument parseXlsx(Path path) throws IOException {
-        List<String> sharedStrings = new ArrayList<>();
-        StringBuilder textBuilder = new StringBuilder(128 * 1024);
-
-        try (java.util.zip.ZipFile zip = new java.util.zip.ZipFile(path.toFile())) {
-            // 1. Read sharedStrings.xml if present
-            java.util.zip.ZipEntry sstEntry = zip.getEntry("xl/sharedStrings.xml");
-            if (sstEntry != null) {
-                try (InputStream is = zip.getInputStream(sstEntry)) {
-                    javax.xml.stream.XMLStreamReader reader = XML_FACTORY.createXMLStreamReader(is);
-
-                    StringBuilder currentText = null;
-                    while (reader.hasNext()) {
-                        int event = reader.next();
-                        if (event == javax.xml.stream.XMLStreamConstants.START_ELEMENT) {
-                            if ("t".equals(reader.getLocalName())) {
-                                currentText = new StringBuilder();
-                            }
-                        } else if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS) {
-                            if (currentText != null) {
-                                currentText.append(reader.getText());
-                            }
-                        } else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) {
-                            if ("t".equals(reader.getLocalName())) {
-                                if (currentText != null) {
-                                    sharedStrings.add(currentText.toString());
-                                    currentText = null;
-                                }
-                            }
-                        }
-                    }
-                } catch (Exception e) {
-                    throw new IOException("Failed parsing XLSX shared strings: " + e.getMessage(), e);
-                }
-            }
-
-            // 2. Read all sheets (sheet1.xml, sheet2.xml, etc.)
-            java.util.Enumeration<? extends java.util.zip.ZipEntry> entries = zip.entries();
-            while (entries.hasMoreElements()) {
-                java.util.zip.ZipEntry entry = entries.nextElement();
-                String entryName = entry.getName();
-                if (entryName.startsWith("xl/worksheets/sheet") && entryName.endsWith(".xml")) {
-                    try (InputStream is = zip.getInputStream(entry)) {
-                        javax.xml.stream.XMLStreamReader reader = XML_FACTORY.createXMLStreamReader(is);
-
-                        String cellType = null;
-                        StringBuilder cellVal = null;
-
-                        while (reader.hasNext()) {
-                            int event = reader.next();
-                            if (event == javax.xml.stream.XMLStreamConstants.START_ELEMENT) {
-                                String name = reader.getLocalName();
-                                if ("c".equals(name)) {
-                                    cellType = reader.getAttributeValue(null, "t");
-                                } else if ("v".equals(name)) {
-                                    cellVal = new StringBuilder();
-                                }
-                            } else if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS) {
-                                if (cellVal != null) {
-                                    cellVal.append(reader.getText());
-                                }
-                            } else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) {
-                                String name = reader.getLocalName();
-                                if ("v".equals(name)) {
-                                    if (cellVal != null) {
-                                        String rawVal = cellVal.toString().trim();
-                                        if ("s".equals(cellType)) {
-                                            try {
-                                                int idx = Integer.parseInt(rawVal);
-                                                if (idx >= 0 && idx < sharedStrings.size()) {
-                                                    textBuilder.append(sharedStrings.get(idx)).append("\t");
-                                                }
-                                            } catch (NumberFormatException ignored) {}
-                                        } else if (!rawVal.isEmpty()) {
-                                            textBuilder.append(rawVal).append("\t");
-                                        }
-                                        cellVal = null;
-                                    }
-                                } else if ("row".equals(name)) {
-                                    textBuilder.append("\n");
-                                }
-                            }
-                        }
-                    } catch (Exception e) {
-                        throw new IOException("Failed parsing XLSX worksheet " + entryName + ": " + e.getMessage(), e);
-                    }
-                }
-            }
-        }
-
-        String normalized = normalize(textBuilder.toString(), "text/plain");
+        String raw = XlsxStreamingParser.extractText(path);
+        String normalized = normalize(raw, "text/plain");
         return new ParsedDocument("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", normalized);
     }
 
     private ParsedDocument parseImageOcr(Path path) throws IOException {
         try {
-            fastocr.FastOCR ocr = new fastocr.FastOCR("en");
+            FastOCR ocr = new FastOCR("en");
             String text = ocr.read(path.toAbsolutePath().toString());
             String normalized = normalize(text, "text/plain");
             return new ParsedDocument("image/ocr", normalized);
@@ -284,7 +183,7 @@ private ParsedDocument parseImageOcr(Path path) throws IOException {
     private ParsedDocument parsePdf(Path path) throws IOException {
         try (PDDocument document = Loader.loadPDF(path.toFile())) {
             VisualParagraphPDFTextStripper stripper = new VisualParagraphPDFTextStripper();
-            stripper.writeText(document, java.io.Writer.nullWriter());
+            stripper.writeText(document, Writer.nullWriter());
             String raw = stripper.buildVisualText();
             String normalized = normalize(raw, "application/pdf");
             return new ParsedDocument("application/pdf", normalized);

src/main/java/fastcontentparse/ParsedDocument.java

@@ -1,18 +1,37 @@
 package fastcontentparse;
 
+/**
+ * Immutable value container representing the result of parsing a document or text buffer.
+ */
 public class ParsedDocument {
     private final String type;
     private final String text;
 
+    /**
+     * Constructs a parsed document entity.
+     *
+     * @param type detected or explicit MIME type
+     * @param text normalized textual content
+     */
     public ParsedDocument(String type, String text) {
         this.type = type;
         this.text = text == null ? "" : text;
     }
 
+    /**
+     * Returns the MIME type of the parsed document (e.g., {@code "application/pdf"}).
+     *
+     * @return MIME type string
+     */
     public String getType() {
         return type;
     }
 
+    /**
+     * Returns the normalized textual content of the parsed document.
+     *
+     * @return clean, whitespace-normalized UTF-8 text
+     */
     public String getText() {
         return text;
     }

src/main/java/fastcontentparse/RtfStripper.java

@@ -0,0 +1,57 @@
+package fastcontentparse;
+
+import fastregex.FastRegex;
+
+/**
+ * Lightweight, single-pass zero-regex RTF lexer and control word stripper.
+ */
+final class RtfStripper {
+
+    private RtfStripper() {
+    }
+
+    /**
+     * Strips RTF group brackets, control words, and symbols from raw RTF text.
+     *
+     * @param input raw RTF text
+     * @return clean extracted text with normalized whitespace
+     */
+    static String strip(String input) {
+        if (input == null || input.isEmpty()) {
+            return "";
+        }
+
+        StringBuilder sb = new StringBuilder(input.length());
+        int i = 0;
+        final int len = input.length();
+
+        while (i < len) {
+            char c = input.charAt(i);
+
+            if (c == '{' || c == '}') {
+                i++;
+                continue;
+            }
+
+            if (c == '\\') {
+                i++;
+                if (i >= len) break;
+
+                char next = input.charAt(i);
+                if (Character.isLetter(next)) {
+                    while (i < len && Character.isLetter(input.charAt(i))) i++;
+                    while (i < len && (Character.isDigit(input.charAt(i)) || input.charAt(i) == '-')) i++;
+                    if (i < len && input.charAt(i) == ' ') i++;
+                } else {
+                    i++;
+                }
+                continue;
+            }
+
+            sb.append(c);
+            i++;
+        }
+
+        return FastRegex.normalizeWhitespace(sb.toString());
+    }
+}