refactor: enhance parsing efficiency with improved field and row estimation (#45)

* refactor: enhance parsing efficiency with improved field and row estimation

* chore: fix lint error

Author: nnnkkk7

field_parser.go

@@ -110,6 +110,19 @@ func parseBuffer(buf []byte, sr *scanResult) *parseResult {
 	// Check if pool capacity is sufficient, otherwise reallocate
 	estimatedFields := len(buf) / avgFieldLenEstimate
 	estimatedRows := len(buf) / avgRowLenEstimate
+	if sr != nil {
+		// Use delimiter counts from scan to size more accurately.
+		if len(buf) > 0 {
+			countFields := sr.separatorCount + sr.newlineCount + 1
+			if countFields > estimatedFields {
+				estimatedFields = countFields
+			}
+			countRows := sr.newlineCount + 1
+			if countRows > estimatedRows {
+				estimatedRows = countRows
+			}
+		}
+	}
 	if cap(result.fields) < estimatedFields {
 		result.fields = make([]fieldInfo, 0, estimatedFields)
 	}

parse.go

@@ -22,6 +22,8 @@ func ParseBytes(data []byte, comma rune) ([][]string, error) {
 
 	// Release parseResult back to pool
 	releaseParseResult(pr)
+	// Release scanResult back to pool
+	releaseScanResult(sr)
 
 	return records, nil
 }
@@ -41,6 +43,7 @@ func ParseBytesStreaming(data []byte, comma rune, callback func([]string) error)
 	// Parse: Extract fields and rows from scan result
 	pr := parseBuffer(data, sr)
 	defer releaseParseResult(pr)
+	defer releaseScanResult(sr)
 
 	if pr == nil || len(pr.rows) == 0 {
 		return nil

reader.go

@@ -66,6 +66,7 @@ type Reader struct {
 	nonCommentRecordCount int          // Count of non-comment records returned (for O(1) first record detection)
 	initialized           bool         // Whether scan/parse have been run
 	hasQuotes             bool         // True if input contains any quote characters (for fast path)
+	chunkHasQuote         []bool       // Per-chunk quote presence (for validation fast path)
 
 	// Extended options (set via NewReaderWithOptions)
 	skipBOM      bool  // Skip UTF-8 BOM if present
@@ -257,6 +258,17 @@ func (r *Reader) initialize() error {
 
 	// Copy hasQuotes flag for fast path optimization
 	r.hasQuotes = r.scanResult.hasQuotes
+	// Copy per-chunk quote presence for validation fast path
+	if len(r.scanResult.chunkHasQuote) > 0 {
+		if cap(r.chunkHasQuote) < len(r.scanResult.chunkHasQuote) {
+			r.chunkHasQuote = make([]bool, len(r.scanResult.chunkHasQuote))
+		} else {
+			r.chunkHasQuote = r.chunkHasQuote[:len(r.scanResult.chunkHasQuote)]
+		}
+		copy(r.chunkHasQuote, r.scanResult.chunkHasQuote)
+	} else {
+		r.chunkHasQuote = nil
+	}
 
 	// Parse: extract fields and rows from scan result
 	// Note: parseBuffer already calls postProcessFields internally
@@ -278,6 +290,13 @@ func (r *Reader) initialize() error {
 // Because ReadAll is defined to read until EOF, it does not
 // treat end of file as an error to be reported.
 func (r *Reader) ReadAll() (records [][]string, err error) {
+	if !r.initialized {
+		if err := r.initialize(); err != nil {
+			return nil, err
+		}
+	}
+	// Defer allocation until we actually have a record
+	// This ensures empty input returns nil (matching encoding/csv behavior)
 	for {
 		record, err := r.Read()
 		if err == io.EOF {
@@ -286,6 +305,9 @@ func (r *Reader) ReadAll() (records [][]string, err error) {
 		if err != nil {
 			return records, err
 		}
+		if records == nil && r.parseResult != nil {
+			records = make([][]string, 0, len(r.parseResult.rows))
+		}
 		records = append(records, record)
 	}
 }
@@ -323,6 +345,7 @@ func (r *Reader) Release() {
 	r.lastRecord = nil
 	r.recordBuffer = nil
 	r.fieldEnds = nil
+	r.chunkHasQuote = nil
 }
 
 // ReaderOptions contains extended configuration options for [Reader].

record_builder.go

@@ -68,9 +68,11 @@ func (r *Reader) buildRecordWithValidation(row rowInfo, rowIdx int) ([]string, e
 
 		// Validate quotes unless LazyQuotes is enabled or no quotes exist in input
 		if !r.LazyQuotes && r.hasQuotes {
-			if err := r.validateFieldQuotes(rawStart, rawEnd, row.lineNum); err != nil {
-				// Build partial record from accumulated content
-				return r.buildPartialRecord(i), err
+			if r.fieldMayContainQuote(rawStart, rawEnd) {
+				if err := r.validateFieldQuotesWithField(field, rawStart, rawEnd, row.lineNum); err != nil {
+					// Build partial record from accumulated content
+					return r.buildPartialRecord(i), err
+				}
 			}
 		}
 

simd_scanner.go

@@ -94,15 +94,18 @@ type scanResult struct {
 	separatorMasks []uint64 // Separator masks per chunk
 	newlineMasks   []uint64 // Newline masks per chunk (CRLF normalized)
 	chunkHasDQ     []bool   // Per-chunk flag: true if chunk contains escaped double quotes
+	chunkHasQuote  []bool   // Per-chunk flag: true if chunk contains any quote
 	hasQuotes      bool     // True if any quote characters exist in input
 	finalQuoted    uint64   // Final quote state
 	chunkCount     int      // Number of processed chunks
 	lastChunkBits  int      // Valid bits in last chunk (if < 64)
+	separatorCount int      // Total separators (after quote invalidation)
+	newlineCount   int      // Total newlines (after quote invalidation)
 }
 
 // scanResultPoolCapacity is the pre-allocated slice capacity for pooled scanResult objects.
-// 1024 chunks = ~64KB input (1024 * 64 bytes per chunk).
-const scanResultPoolCapacity = 1024
+// 64 chunks = ~4KB input (64 * 64 bytes per chunk) - small default, grows as needed.
+const scanResultPoolCapacity = 64
 
 // scanResultPool provides reusable scanResult objects to reduce allocations.
 var scanResultPool = sync.Pool{
@@ -112,6 +115,7 @@ var scanResultPool = sync.Pool{
 			separatorMasks: make([]uint64, 0, scanResultPoolCapacity),
 			newlineMasks:   make([]uint64, 0, scanResultPoolCapacity),
 			chunkHasDQ:     make([]bool, 0, scanResultPoolCapacity),
+			chunkHasQuote:  make([]bool, 0, scanResultPoolCapacity),
 		}
 	},
 }
@@ -124,10 +128,15 @@ func (sr *scanResult) reset() {
 	if cap(sr.chunkHasDQ) > 0 {
 		sr.chunkHasDQ = sr.chunkHasDQ[:0]
 	}
+	if cap(sr.chunkHasQuote) > 0 {
+		sr.chunkHasQuote = sr.chunkHasQuote[:0]
+	}
 	sr.hasQuotes = false
 	sr.finalQuoted = 0
 	sr.chunkCount = 0
 	sr.lastChunkBits = 0
+	sr.separatorCount = 0
+	sr.newlineCount = 0
 }
 
 // releaseScanResult returns a scanResult to the pool for reuse.
@@ -170,6 +179,11 @@ func generateMasksAVX512(data []byte, separator byte) (quote, sep, cr, nl uint64
 	crCmp := archsimd.BroadcastInt8x32('\r')
 	nlCmp := archsimd.BroadcastInt8x32('\n')
 
+	return generateMasksAVX512WithCmp(data, quoteCmp, sepCmp, crCmp, nlCmp)
+}
+
+// generateMasksAVX512WithCmp is an AVX-512 mask generator that reuses pre-broadcasted comparators.
+func generateMasksAVX512WithCmp(data []byte, quoteCmp, sepCmp, crCmp, nlCmp archsimd.Int8x32) (quote, sep, cr, nl uint64) {
 	// Process low simdHalfChunk bytes (positions 0-31)
 	// Precondition: data is at least simdChunkSize bytes (guaranteed by caller)
 	low := archsimd.LoadInt8x32((*[simdHalfChunk]int8)(unsafe.Pointer(&data[0])))
@@ -350,6 +364,10 @@ func scanBuffer(buf []byte, separatorChar byte) *scanResult {
 	result.reset()
 	result.chunkCount = chunkCount
 
+	// NOTE: Pre-broadcasting AVX-512 comparators was attempted but removed because
+	// declaring archsimd.Int8x32 variables causes Go to emit AVX zeroing instructions
+	// even before the conditional check, causing SIGILL on CPUs without AVX support.
+
 	// Pre-size all mask slices to chunkCount for index-based assignment (avoids append overhead)
 	// When capacity is insufficient, grow by 2x to reduce future reallocations
 	if cap(result.quoteMasks) < chunkCount {
@@ -392,6 +410,18 @@ func scanBuffer(buf []byte, separatorChar byte) *scanResult {
 			result.chunkHasDQ[i] = false
 		}
 	}
+	if cap(result.chunkHasQuote) < chunkCount {
+		newCap := chunkCount
+		if newCap < cap(result.chunkHasQuote)*2 {
+			newCap = cap(result.chunkHasQuote) * 2
+		}
+		result.chunkHasQuote = make([]bool, chunkCount, newCap)
+	} else {
+		result.chunkHasQuote = result.chunkHasQuote[:chunkCount]
+		for i := range result.chunkHasQuote {
+			result.chunkHasQuote[i] = false
+		}
+	}
 
 	state := scanState{}
 
@@ -500,13 +530,22 @@ func scanBuffer(buf []byte, separatorChar byte) *scanResult {
 		// Track if any quotes exist in the input (for fast path optimization)
 		if quoteMaskOut != 0 {
 			result.hasQuotes = true
+			result.chunkHasQuote[chunkIdx] = true
 		}
 
 		// Record chunks that have double quotes (using bool array instead of []int)
 		if hasDoubleQuote {
 			result.chunkHasDQ[chunkIdx] = true
 		}
 
+		// Accumulate counts for preallocation sizing
+		if sepMaskOut != 0 {
+			result.separatorCount += bits.OnesCount64(sepMaskOut)
+		}
+		if newlineMaskOut != 0 {
+			result.newlineCount += bits.OnesCount64(newlineMaskOut)
+		}
+
 		// Slide masks: current = next, compute new next for chunkIdx+2
 		curMasks = nextMasks
 		curValidBits = simdChunkSize // next chunk was full unless it's the last

validation.go

@@ -2,9 +2,39 @@
 
 package simdcsv
 
+// fieldMayContainQuote returns true if any chunk overlapped by the field contains a quote.
+// This allows skipping validation when it's impossible for the field to contain quotes.
+func (r *Reader) fieldMayContainQuote(rawStart, rawEnd uint64) bool {
+	if len(r.chunkHasQuote) == 0 {
+		return true
+	}
+	if rawEnd <= rawStart {
+		return false
+	}
+	startChunk := int(rawStart / simdChunkSize)   //nolint:gosec // G115: rawStart bounded by buffer size (max 2GB)
+	endChunk := int((rawEnd - 1) / simdChunkSize) //nolint:gosec // G115: rawEnd bounded by buffer size (max 2GB)
+	if startChunk < 0 {
+		return true
+	}
+	if endChunk >= len(r.chunkHasQuote) {
+		endChunk = len(r.chunkHasQuote) - 1
+	}
+	for i := startChunk; i <= endChunk; i++ {
+		if r.chunkHasQuote[i] {
+			return true
+		}
+	}
+	return false
+}
+
 // validateFieldQuotes validates quote usage in a field.
 // This is the main entry point that dispatches to quoted or unquoted validation.
 func (r *Reader) validateFieldQuotes(rawStart, rawEnd uint64, lineNum int) error {
+	return r.validateFieldQuotesWithField(fieldInfo{}, rawStart, rawEnd, lineNum)
+}
+
+// validateFieldQuotesWithField validates quote usage in a field using field metadata when available.
+func (r *Reader) validateFieldQuotesWithField(field fieldInfo, rawStart, rawEnd uint64, lineNum int) error {
 	if rawStart >= uint64(len(r.rawBuffer)) || rawEnd > uint64(len(r.rawBuffer)) || rawStart >= rawEnd {
 		return nil
 	}
@@ -20,12 +50,71 @@ func (r *Reader) validateFieldQuotes(rawStart, rawEnd uint64, lineNum int) error
 		// Adjust raw to start from the quote for validation
 		adjustedRaw := raw[quoteOffset:]
 		adjustedStart := rawStart + uint64(quoteOffset) //nolint:gosec // G115: quoteOffset is always non-negative from isQuotedFieldStart
+		// Fast path: use parsed field metadata when opening quote is at start
+		if quoteOffset == 0 && field.rawEndDelta != 0 {
+			if err := r.validateQuotedFieldFast(adjustedRaw, adjustedStart, field, lineNum); err != nil {
+				return err
+			}
+			return nil
+		}
 		return r.validateQuotedField(adjustedRaw, adjustedStart, lineNum)
 	}
 
 	return r.validateUnquotedField(raw, rawStart, lineNum)
 }
 
+// validateQuotedFieldFast validates a quoted field using parsed metadata to avoid rescanning.
+// raw must start with the opening quote.
+func (r *Reader) validateQuotedFieldFast(raw []byte, rawStart uint64, field fieldInfo, lineNum int) error {
+	if len(raw) < 2 || raw[0] != '"' {
+		return r.validateQuotedField(raw, rawStart, lineNum)
+	}
+
+	closingIdx := int(field.length) + 1
+	if closingIdx >= len(raw) || raw[closingIdx] != '"' {
+		// No closing quote found at expected position
+		return &ParseError{
+			StartLine: lineNum,
+			Line:      lineNum,
+			Column:    int(rawStart) + len(raw), //nolint:gosec // G115: rawStart bounded by buffer size
+			Err:       ErrQuote,
+		}
+	}
+
+	switch field.rawEndDelta {
+	case 1:
+		// Closing quote should be immediately before delimiter/EOF.
+		if closingIdx+1 != len(raw) {
+			return &ParseError{
+				StartLine: lineNum,
+				Line:      lineNum,
+				Column:    int(rawStart) + closingIdx + 2, //nolint:gosec // G115: rawStart bounded by buffer size
+				Err:       ErrQuote,
+			}
+		}
+	case 2:
+		// CRLF: raw includes trailing \r before the delimiter LF.
+		if closingIdx+2 != len(raw) || raw[closingIdx+1] != '\r' {
+			return &ParseError{
+				StartLine: lineNum,
+				Line:      lineNum,
+				Column:    int(rawStart) + closingIdx + 2, //nolint:gosec // G115: rawStart bounded by buffer size
+				Err:       ErrQuote,
+			}
+		}
+	default:
+		// Extra data after closing quote (invalid)
+		return &ParseError{
+			StartLine: lineNum,
+			Line:      lineNum,
+			Column:    int(rawStart) + closingIdx + 2, //nolint:gosec // G115: rawStart bounded by buffer size
+			Err:       ErrQuote,
+		}
+	}
+
+	return nil
+}
+
 // validateQuotedField validates a field that starts with a quote.
 // raw should start with the opening quote.
 func (r *Reader) validateQuotedField(raw []byte, rawStart uint64, lineNum int) error {