refactor: enhance field parsing by adding quote detection and optimization flags (#72)

* refactor: enhance field parsing by adding quote detection and optimization flags

* refactor: optimize quote detection in scalar and SIMD processing

* refactor: clean up field flag definitions and remove unused quote detection functions

Author: nnnkkk7

field_parser.go

@@ -39,6 +39,7 @@ type parserState struct {
 	quoteAdjust      uint64 // bytes to skip for opening quote (0 or 1)
 	lastSepOrNewline int64  // last separator/newline position (-1 initially)
 	lastClosingQuote int64  // last closing quote position (-1 if none)
+	sawQuote         bool   // true if quote was seen in current field (for validation optimization)
 }
 
 // newParserState creates an initialized parser state.
@@ -67,6 +68,7 @@ func (s *parserState) resetForNextField(delimiterPos uint64) {
 	s.quoteAdjust = 0
 	s.lastSepOrNewline = int64(delimiterPos)
 	s.lastClosingQuote = -1
+	s.sawQuote = false
 }
 
 // =============================================================================
@@ -122,12 +124,13 @@ type fieldInfo struct {
 	start       uint32 // content start offset (after opening quote if quoted)
 	length      uint32 // content length (excluding quotes)
 	rawEndDelta uint8  // delta from start+length to raw end position
-	flags       uint8  // bit0: needsUnescape, bit1: isQuoted
+	flags       uint8  // bit0: needsUnescape, bit1: isQuoted, bit2: containsQuote
 }
 
 const (
 	fieldFlagNeedsUnescape = 1 << 0
 	fieldFlagIsQuoted      = 1 << 1
+	fieldFlagContainsQuote = 1 << 2 // field contains quote character (for validation optimization)
 )
 
 // rawStart returns the raw start position (including opening quote if quoted).
@@ -144,11 +147,14 @@ func (f *fieldInfo) rawEnd() uint32 {
 }
 
 // newFieldInfo creates a fieldInfo from parsed boundaries.
-func newFieldInfo(start, length uint64, rawEndDelta uint8, isQuoted bool) fieldInfo {
+func newFieldInfo(start, length uint64, rawEndDelta uint8, isQuoted, containsQuote bool) fieldInfo {
 	var flags uint8
 	if isQuoted {
 		flags = fieldFlagIsQuoted
 	}
+	if containsQuote {
+		flags |= fieldFlagContainsQuote
+	}
 	return fieldInfo{
 		start:       uint32(start),
 		length:      uint32(length),
@@ -171,6 +177,12 @@ func (f *fieldInfo) needsUnescape() bool {
 	return f.flags&fieldFlagNeedsUnescape != 0
 }
 
+// containsQuote returns whether the field contains any quote characters.
+// Used for validation optimization - fields without quotes don't need quote validation.
+func (f *fieldInfo) containsQuote() bool {
+	return f.flags&fieldFlagContainsQuote != 0
+}
+
 // rowInfo holds row metadata.
 type rowInfo struct {
 	firstField int // index of first field in parseResult.fields
@@ -387,6 +399,7 @@ func classifyEvent(bit, quoteMask, sepMask uint64) eventType {
 
 // handleQuoteEvent processes a quote character, toggling the quoted state.
 func handleQuoteEvent(absPos uint64, state *parserState) {
+	state.sawQuote = true // Mark that this field contains a quote
 	if state.quoted {
 		state.exitQuotedState(absPos)
 	} else {
@@ -430,6 +443,7 @@ func skipBlankLine(state *parserState, absPos uint64, lineNum *int) {
 	state.fieldStart = absPos + 1
 	state.quoteAdjust = 0
 	state.lastClosingQuote = -1
+	state.sawQuote = false
 	(*lineNum)++
 }
 
@@ -441,7 +455,8 @@ func skipBlankLine(state *parserState, absPos uint64, lineNum *int) {
 // For newline delimiters (isNewline=true), excludes trailing CR from CRLF sequences.
 func recordField(buf []byte, absPos uint64, state *parserState, result *parseResult, isNewline bool) {
 	bounds := computeFieldBounds(buf, absPos, state, isNewline)
-	result.fields = append(result.fields, newFieldInfo(bounds.start, bounds.length, bounds.rawEndDelta, bounds.isQuoted))
+	containsQuote := state.sawQuote
+	result.fields = append(result.fields, newFieldInfo(bounds.start, bounds.length, bounds.rawEndDelta, bounds.isQuoted, containsQuote))
 	state.resetForNextField(absPos)
 }
 
@@ -544,8 +559,9 @@ func finalizeLastField(buf []byte, state *parserState, result *parseResult, rowF
 	fieldLen := computeFieldLength(bufLen, start, state)
 	rawEndDelta := computeRawEndDelta(bufLen, start, fieldLen)
 	isQuoted := state.quoteAdjust > 0
+	containsQuote := state.sawQuote
 
-	result.fields = append(result.fields, newFieldInfo(start, fieldLen, rawEndDelta, isQuoted))
+	result.fields = append(result.fields, newFieldInfo(start, fieldLen, rawEndDelta, isQuoted, containsQuote))
 	result.rows = append(result.rows, rowInfo{
 		firstField: rowFirstField,
 		fieldCount: len(result.fields) - rowFirstField,

quote.go

@@ -3,6 +3,7 @@
 package simdcsv
 
 import (
+	"bytes"
 	"math/bits"
 
 	"simd/archsimd"
@@ -78,15 +79,17 @@ func findClosingQuote(data []byte, startAfterOpenQuote int) int {
 
 // findClosingQuoteScalar finds the closing quote using scalar operations.
 func findClosingQuoteScalar(data []byte, startAfterOpenQuote int) int {
-	for i := startAfterOpenQuote; i < len(data); i++ {
-		if data[i] != '"' {
-			continue
+	for i := startAfterOpenQuote; i < len(data); {
+		next := bytes.IndexByte(data[i:], '"')
+		if next == -1 {
+			return -1
 		}
-		if isEscapedQuote(data, i) {
-			i++ // Skip second quote of escape sequence (loop increments again)
+		pos := i + next
+		if isEscapedQuote(data, pos) {
+			i = pos + 2
 			continue
 		}
-		return i
+		return pos
 	}
 	return -1
 }
@@ -125,6 +128,21 @@ func findClosingQuoteSIMD(data []byte, startAfterOpenQuote int) int {
 // processQuoteMask processes quote positions in a SIMD chunk mask.
 // Returns (closingQuoteIdx, newPosition, shouldExitLoop).
 func processQuoteMask(data []byte, chunkStart int, mask uint64) (int, int, bool) {
+	// Fast path: no adjacent quotes in this chunk, so first quote closes.
+	if mask&(mask<<1) == 0 {
+		pos := bits.TrailingZeros64(mask)
+		if pos == simdChunkSize-1 {
+			newPos := chunkStart + simdChunkSize
+			if newPos < len(data) && data[newPos] == '"' {
+				// Boundary double quote → skip both
+				return -1, newPos + 1, false
+			}
+			// Closing quote at boundary
+			return chunkStart + pos, chunkStart, false
+		}
+		return chunkStart + pos, chunkStart, false
+	}
+
 	for mask != 0 {
 		pos := bits.TrailingZeros64(mask)
 

record_builder.go

@@ -230,11 +230,12 @@ func (r *Reader) validateFieldIfNeeded(field fieldInfo, lineNum int) error {
 		return nil
 	}
 
-	rawStart, rawEnd := uint64(field.rawStart()), uint64(field.rawEnd())
-	if !r.fieldMayContainQuote(rawStart, rawEnd) {
+	// Fast path: field doesn't contain any quotes (set during parsing)
+	if !field.containsQuote() {
 		return nil
 	}
 
+	rawStart, rawEnd := uint64(field.rawStart()), uint64(field.rawEnd())
 	return r.validateFieldQuotesWithField(field, rawStart, rawEnd, lineNum)
 }
 

simd_scanner.go

@@ -337,28 +337,46 @@ func normalizeCRLF(crMask, nlMask, nextNlMask uint64, validBits int) uint64 {
 // separators inside quoted regions. Detects escaped double quotes ("") including
 // those spanning chunk boundaries.
 func processQuotesAndSeparators(quoteMask, sepMask, nextQuoteMask uint64, state *scanState) (quoteMaskOut, sepMaskOut uint64, hasDoubleQuote, boundaryDoubleQuote bool) {
-	quoteMaskOut = quoteMask
-	workQuote := quoteMask
 	quoted := state.quoted
 	initialQuoted := quoted
 
-	// Step 1: Process quotes to detect and remove double quotes
+	// Fast path: no quotes in this chunk
+	if quoteMask == 0 {
+		if quoted == 0 {
+			return 0, sepMask, false, false
+		}
+		// All separators are inside quoted region
+		return 0, 0, false, false
+	}
+
+	quoteMaskOut = quoteMask
+
+	// Pre-detect adjacent quote pairs (potential double quotes when inside quoted region)
+	// adjacentPairs has bit set at the LEFT position of each adjacent pair
+	adjacentPairs := quoteMask & (quoteMask >> 1)
+
+	// Pre-check for boundary double quote (quote at pos 63 with quote at pos 0 of next chunk)
+	const lastBit = uint64(1) << 63
+	boundaryCandidate := quoteMask&lastBit != 0 && nextQuoteMask&1 != 0
+
+	workQuote := quoteMask
 	for workQuote != 0 {
 		pos := bits.TrailingZeros64(workQuote)
 		bit := uint64(1) << pos
 
 		if quoted != 0 {
 			// Inside quotes: check for escaped double quote
-			if pos == simdChunkSize-1 && nextQuoteMask&1 != 0 {
+			if adjacentPairs&bit != 0 {
+				// Adjacent double quote - remove both quotes from output
+				nextBit := bit << 1
+				quoteMaskOut &^= bit | nextBit
+				hasDoubleQuote = true
+				workQuote &^= nextBit // Skip next quote
+			} else if pos == 63 && boundaryCandidate {
 				// Boundary double quote
-				quoteMaskOut &^= uint64(1) << (simdChunkSize - 1)
+				quoteMaskOut &^= lastBit
 				hasDoubleQuote = true
 				boundaryDoubleQuote = true
-			} else if pos < simdChunkSize-1 && workQuote&(uint64(1)<<(pos+1)) != 0 {
-				// Adjacent double quote
-				quoteMaskOut &^= uint64(3) << pos
-				hasDoubleQuote = true
-				workQuote &^= uint64(1) << (pos + 1)
 			} else {
 				// Closing quote
 				quoted = 0
@@ -372,7 +390,7 @@ func processQuotesAndSeparators(quoteMask, sepMask, nextQuoteMask uint64, state
 
 	state.quoted = quoted
 
-	// Step 2: Invalidate separators using prefix XOR on clean quote mask
+	// Invalidate separators using prefix XOR on cleaned quote mask
 	inQuote := quoteMaskOut
 	inQuote ^= inQuote << 1
 	inQuote ^= inQuote << 2

validation.go

@@ -30,37 +30,6 @@ func (p validationPolicy) shouldUseMetadata(field fieldInfo) bool {
 	return field.flags&fieldFlagIsQuoted != 0 && !p.trimLeadingSpace
 }
 
-// =============================================================================
-// Chunk-Level Quote Detection - Fast path optimization
-// =============================================================================
-
-// fieldMayContainQuote reports whether any chunk overlapped by the field contains a quote.
-// Returns true conservatively when chunk data is unavailable.
-func (r *Reader) fieldMayContainQuote(rawStart, rawEnd uint64) bool {
-	if len(r.state.chunkHasQuote) == 0 {
-		return true // Conservative: assume quotes when no chunk data
-	}
-	if rawEnd <= rawStart {
-		return false // Empty range contains nothing
-	}
-
-	startChunk := int(rawStart / simdChunkSize)   //nolint:gosec // G115
-	endChunk := int((rawEnd - 1) / simdChunkSize) //nolint:gosec // G115
-	endChunk = min(endChunk, len(r.state.chunkHasQuote)-1)
-
-	return anyChunkHasQuote(r.state.chunkHasQuote, startChunk, endChunk)
-}
-
-// anyChunkHasQuote checks if any chunk in the range [start, end] contains a quote.
-func anyChunkHasQuote(chunks []bool, start, end int) bool {
-	for i := start; i <= end; i++ {
-		if chunks[i] {
-			return true
-		}
-	}
-	return false
-}
-
 // =============================================================================
 // Field Extraction - Mechanism for accessing raw field data
 // =============================================================================