feat: use PCLMULQDQ

benchmark_test.go

@@ -570,7 +570,7 @@ func BenchmarkGenerateMasks(b *testing.B) {
 	copy(data, []byte(`"field1","field2","field3","field4","field5","field6","fie"`))
 
 	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
+	for b.Loop() {
 		generateMasks(data, ',')
 	}
 }
@@ -590,7 +590,7 @@ func BenchmarkGenerateMasksPadded(b *testing.B) {
 			}
 
 			b.ResetTimer()
-			for i := 0; i < b.N; i++ {
+			for b.Loop() {
 				generateMasksPadded(data, ',')
 			}
 		})
@@ -617,7 +617,7 @@ func BenchmarkScanBuffer(b *testing.B) {
 
 			b.ResetTimer()
 			b.SetBytes(int64(size))
-			for i := 0; i < b.N; i++ {
+			for b.Loop() {
 				scanBuffer(data, ',')
 			}
 		})
@@ -665,7 +665,65 @@ func BenchmarkParseBuffer(b *testing.B) {
 	b.ResetTimer()
 	b.ReportAllocs()
 
-	for i := 0; i < b.N; i++ {
+	for b.Loop() {
 		_ = parseBuffer(data, sr)
 	}
 }
+
+// =============================================================================
+// prefixXOR Benchmarks - PCLMULQDQ
+// =============================================================================
+
+func BenchmarkPrefixXOR(b *testing.B) {
+	// Create test masks with varying densities
+	testCases := []struct {
+		name string
+		mask uint64
+	}{
+		{"empty", 0},
+		{"single_bit", 1},
+		{"sparse", 0x0001000100010001}, // few bits set
+		{"medium", 0x5555555555555555}, // alternating bits
+		{"dense", 0xFFFFFFFFFFFFFFFF},  // all bits set
+		{"realistic", 0b0100010001000100010001000100010001000100010001000100010001000100}, // quote-like pattern
+	}
+
+	for _, tc := range testCases {
+		b.Run(tc.name, func(b *testing.B) {
+			for b.Loop() {
+				_ = prefixXOR(tc.mask)
+			}
+		})
+	}
+}
+
+// BenchmarkPrefixXORThroughput measures throughput with sequential masks
+func BenchmarkPrefixXORThroughput(b *testing.B) {
+	// Pre-generate masks to avoid setup overhead
+	masks := make([]uint64, 1024)
+	state := uint64(0xDEADBEEFCAFEBABE)
+	for i := range masks {
+		state ^= state << 13
+		state ^= state >> 7
+		state ^= state << 17
+		masks[i] = state
+	}
+
+	idx := 0
+	for b.Loop() {
+		_ = prefixXOR(masks[idx%len(masks)])
+		idx++
+	}
+}
+
+// BenchmarkPrefixXORLatencyChain measures latency when each call depends on previous
+func BenchmarkPrefixXORLatencyChain(b *testing.B) {
+	mask := uint64(0x5555555555555555)
+	for b.Loop() {
+		mask = prefixXOR(mask)
+	}
+	// Prevent compiler from optimizing away
+	if mask == 0 {
+		b.Fatal("unexpected zero")
+	}
+}

simd_scanner.go

@@ -28,6 +28,9 @@ var (
 	cachedCrCmp    archsimd.Int8x64
 	cachedNlCmp    archsimd.Int8x64
 	cachedSepCmp   [cachedSepCmpCount]archsimd.Int8x64
+
+	// PCLMULQDQ cached value: all-ones for carryless multiplication
+	cachedAllOnes archsimd.Uint64x2
 )
 
 // SIMD processing constants.
@@ -50,6 +53,13 @@ func init() {
 		cachedQuoteCmp = cachedSepCmp['"']
 		cachedCrCmp = cachedSepCmp['\r']
 		cachedNlCmp = cachedSepCmp['\n']
+
+		// Pre-load all-ones value for carryless multiplication (PCLMULQDQ)
+		// Used in prefixXOR: mask × 0xFFFFFFFFFFFFFFFF computes prefix XOR
+		cachedAllOnes = archsimd.LoadUint64x2(&[2]uint64{
+			0xFFFFFFFFFFFFFFFF,
+			0xFFFFFFFFFFFFFFFF,
+		})
 	}
 }
 
@@ -63,6 +73,42 @@ func shouldUseSIMD(dataLen int) bool {
 	return useAVX512 && dataLen >= simdMinThreshold
 }
 
+// =============================================================================
+// Prefix XOR (Quote Region Mask Computation)
+// =============================================================================
+
+// prefixXOR computes the prefix XOR of the input mask.
+// For each bit position i, the result bit is the XOR of bits 0 through i.
+//
+// This is used to convert a quote position mask into an "inside quotes" mask:
+//
+//	input:  0b01001010 (quote positions at 1, 3, 6)
+//	output: 0b11000110 (inside quote regions)
+//
+// When AVX-512 is available, uses PCLMULQDQ instruction for ~3x speedup.
+// Mathematical basis (carryless multiplication in GF(2)):
+//
+//	mask × all_ones = mask × (1 + 2 + 4 + ... + 2^63)
+//	                = mask ^ (mask << 1) ^ (mask << 2) ^ ... ^ (mask << 63)
+//
+// The lower 64 bits of this product give us the prefix XOR.
+func prefixXOR(mask uint64) uint64 {
+	if useAVX512 {
+		// PCLMULQDQ path: ~3-4 instructions
+		maskVec := archsimd.LoadUint64x2(&[2]uint64{mask, 0})
+		result := maskVec.CarrylessMultiply(0, 0, cachedAllOnes)
+		return result.GetElem(0)
+	}
+	// Scalar path: 6 shifts + 6 XORs = 12 instructions
+	mask ^= mask << 1
+	mask ^= mask << 2
+	mask ^= mask << 4
+	mask ^= mask << 8
+	mask ^= mask << 16
+	mask ^= mask << 32
+	return mask
+}
+
 // =============================================================================
 // Core Data Structures
 // =============================================================================
@@ -391,13 +437,8 @@ func processQuotesAndSeparators(quoteMask, sepMask, nextQuoteMask uint64, state
 	state.quoted = quoted
 
 	// Invalidate separators using prefix XOR on cleaned quote mask
-	inQuote := quoteMaskOut
-	inQuote ^= inQuote << 1
-	inQuote ^= inQuote << 2
-	inQuote ^= inQuote << 4
-	inQuote ^= inQuote << 8
-	inQuote ^= inQuote << 16
-	inQuote ^= inQuote << 32
+	// Uses PCLMULQDQ when available for ~3x fewer instructions
+	inQuote := prefixXOR(quoteMaskOut)
 
 	if initialQuoted != 0 {
 		inQuote = ^inQuote
@@ -410,13 +451,8 @@ func processQuotesAndSeparators(quoteMask, sepMask, nextQuoteMask uint64, state
 // invalidateNewlinesInQuotes removes newline bits that are inside quoted regions.
 func invalidateNewlinesInQuotes(quoteMask, newlineMask uint64, state *scanState) uint64 {
 	// Prefix XOR: inQuote[i] = 1 iff positions 0..i have odd number of quotes
-	inQuote := quoteMask
-	inQuote ^= inQuote << 1
-	inQuote ^= inQuote << 2
-	inQuote ^= inQuote << 4
-	inQuote ^= inQuote << 8
-	inQuote ^= inQuote << 16
-	inQuote ^= inQuote << 32
+	// Uses PCLMULQDQ when available for ~3x fewer instructions
+	inQuote := prefixXOR(quoteMask)
 
 	// If we started inside a quoted region, invert the mask
 	if state.quoted != 0 {

simd_scanner_test.go

@@ -1160,3 +1160,165 @@ func popcount(x uint64) int {
 	}
 	return count
 }
+
+// ============================================================================
+// TestPrefixXOR - Test prefix XOR computation (PCLMULQDQ optimization)
+// ============================================================================
+
+func TestPrefixXOR(t *testing.T) {
+	// prefixXOR computes the cumulative XOR: result[i] = XOR of bits 0..i
+	// This is used for quote region detection: bit i is set if there's an
+	// odd number of quotes at positions 0 through i (inclusive).
+	tests := []struct {
+		name  string
+		input uint64
+		want  uint64
+	}{
+		{
+			name:  "empty",
+			input: 0,
+			want:  0, // no quotes = no regions
+		},
+		{
+			name:  "single_bit_0",
+			input: 1, // quote at position 0
+			// All positions 0..63 have odd count (1) → all bits set
+			want: 0xFFFFFFFFFFFFFFFF,
+		},
+		{
+			name:  "single_bit_1",
+			input: 2, // quote at position 1 (0b10)
+			// pos 0: 0 quotes → 0; pos 1+: 1 quote → 1
+			want: 0xFFFFFFFFFFFFFFFE,
+		},
+		{
+			name:  "single_bit_2",
+			input: 4, // quote at position 2 (0b100)
+			// pos 0,1: 0 quotes → 0; pos 2+: 1 quote → 1
+			want: 0xFFFFFFFFFFFFFFFC,
+		},
+		{
+			name:  "two_adjacent_bits",
+			input: 0b11, // quotes at positions 0,1
+			// pos 0: 1 quote → 1; pos 1+: 2 quotes → 0
+			want: 0x0000000000000001,
+		},
+		{
+			name:  "alternating_bits_8bit",
+			input: 0xAA, // 0b10101010 = quotes at positions 1,3,5,7
+			// pos 0: 0→0, pos 1: 1→1, pos 2: 1→1, pos 3: 2→0,
+			// pos 4: 2→0, pos 5: 3→1, pos 6: 3→1, pos 7: 4→0
+			// Low 8 bits: 0b01100110 = 0x66
+			// pos 8+: 4 quotes (even) → 0
+			want: 0x0000000000000066,
+		},
+		{
+			name:  "quote_example",
+			input: 0x4A, // 0b01001010 = quotes at positions 1, 3, 6
+			// pos 0: 0→0, pos 1: 1→1, pos 2: 1→1, pos 3: 2→0,
+			// pos 4: 2→0, pos 5: 2→0, pos 6: 3→1, pos 7+: 3→1 (odd)
+			// Low 8 bits: 0b11000110 = 0xC6
+			// Upper bits: all 1 (odd count continues)
+			want: 0xFFFFFFFFFFFFFFC6,
+		},
+		{
+			name:  "all_ones_8bit",
+			input: 0xFF, // quotes at all positions 0-7
+			// pos 0: 1→1, pos 1: 2→0, pos 2: 3→1, pos 3: 4→0, ...
+			// Pattern: 10101010... = 0x55 for low 8 bits
+			// pos 8+: 8 quotes (even) → 0
+			want: 0x0000000000000055,
+		},
+		{
+			name:  "high_bit_only",
+			input: uint64(1) << 63, // quote at position 63 only
+			// pos 0-62: 0 quotes → 0; pos 63: 1 quote → 1
+			want: 0x8000000000000000,
+		},
+		{
+			name:  "bits_0_and_63",
+			input: 1 | (uint64(1) << 63), // quotes at positions 0 and 63
+			// pos 0: 1→1, pos 1-62: 1→1 (odd), pos 63: 2→0
+			want: 0x7FFFFFFFFFFFFFFF,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := prefixXOR(tt.input)
+			if got != tt.want {
+				t.Errorf("prefixXOR(0x%016x) = 0x%016x, want 0x%016x",
+					tt.input, got, tt.want)
+			}
+		})
+	}
+}
+
+// TestPrefixXORQuoteRegions tests the actual use case: converting quote positions to regions
+func TestPrefixXORQuoteRegions(t *testing.T) {
+	// prefixXOR result[i] = 1 means odd number of quotes at positions 0..i (inclusive)
+	// In CSV parsing context:
+	// - Position i has inQuote[i]=1 if we're at or after an opening quote but not past a closing quote
+	// - Quote chars themselves are considered "in quote" for masking purposes
+	tests := []struct {
+		name          string
+		quotePos      []int // positions where quotes appear
+		wantInQuote   []int // positions where inQuote bit should be 1
+		wantOutQuote  []int // positions where inQuote bit should be 0
+		initialQuoted bool  // if true, we start inside a quoted region
+	}{
+		{
+			name:     "simple_quoted_field",
+			quotePos: []int{0, 5}, // "hello" - quotes at 0 and 5
+			// pos 0: 1 quote → 1; pos 1-4: 1 quote → 1; pos 5: 2 quotes → 0
+			wantInQuote:  []int{0, 1, 2, 3, 4},
+			wantOutQuote: []int{5, 6, 7, 8},
+		},
+		{
+			name:     "two_quoted_fields",
+			quotePos: []int{0, 3, 5, 8}, // "ab","cd" - pattern at 0,3,5,8
+			// pos 0-2: 1 quote → 1; pos 3-4: 2 quotes → 0; pos 5-7: 3 quotes → 1; pos 8+: 4 quotes → 0
+			wantInQuote:  []int{0, 1, 2, 5, 6, 7},
+			wantOutQuote: []int{3, 4, 8, 9, 10},
+		},
+		{
+			name:     "start_inside_quote",
+			quotePos: []int{5}, // closing quote at 5
+			// Without inversion: pos 0-4: 0 quotes → 0; pos 5+: 1 quote → 1
+			// With inversion (initialQuoted=true): pos 0-4: 1; pos 5+: 0
+			wantInQuote:   []int{0, 1, 2, 3, 4},
+			wantOutQuote:  []int{5, 6, 7, 8},
+			initialQuoted: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Build quote mask
+			var quoteMask uint64
+			for _, pos := range tt.quotePos {
+				quoteMask |= uint64(1) << pos
+			}
+
+			// Compute in-quote mask using prefixXOR
+			inQuote := prefixXOR(quoteMask)
+			if tt.initialQuoted {
+				inQuote = ^inQuote
+			}
+
+			// Verify expected in-quote positions
+			for _, pos := range tt.wantInQuote {
+				if inQuote&(uint64(1)<<pos) == 0 {
+					t.Errorf("position %d should be inside quotes (bit=1), but bit is 0. inQuote=0x%016x", pos, inQuote)
+				}
+			}
+
+			// Verify expected out-quote positions
+			for _, pos := range tt.wantOutQuote {
+				if inQuote&(uint64(1)<<pos) != 0 {
+					t.Errorf("position %d should be outside quotes (bit=0), but bit is 1. inQuote=0x%016x", pos, inQuote)
+				}
+			}
+		})
+	}
+}