diff --git a/benchmark_test.go b/benchmark_test.go index 2a40bc6..e52fafc 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -570,7 +570,7 @@ func BenchmarkGenerateMasks(b *testing.B) { copy(data, []byte(`"field1","field2","field3","field4","field5","field6","fie"`)) b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { generateMasks(data, ',') } } @@ -590,7 +590,7 @@ func BenchmarkGenerateMasksPadded(b *testing.B) { } b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { generateMasksPadded(data, ',') } }) @@ -617,7 +617,7 @@ func BenchmarkScanBuffer(b *testing.B) { b.ResetTimer() b.SetBytes(int64(size)) - for i := 0; i < b.N; i++ { + for b.Loop() { scanBuffer(data, ',') } }) @@ -665,7 +665,65 @@ func BenchmarkParseBuffer(b *testing.B) { b.ResetTimer() b.ReportAllocs() - for i := 0; i < b.N; i++ { + for b.Loop() { _ = parseBuffer(data, sr) } } + +// ============================================================================= +// prefixXOR Benchmarks - PCLMULQDQ +// ============================================================================= + +func BenchmarkPrefixXOR(b *testing.B) { + // Create test masks with varying densities + testCases := []struct { + name string + mask uint64 + }{ + {"empty", 0}, + {"single_bit", 1}, + {"sparse", 0x0001000100010001}, // few bits set + {"medium", 0x5555555555555555}, // alternating bits + {"dense", 0xFFFFFFFFFFFFFFFF}, // all bits set + {"realistic", 0b0100010001000100010001000100010001000100010001000100010001000100}, // quote-like pattern + } + + for _, tc := range testCases { + b.Run(tc.name, func(b *testing.B) { + for b.Loop() { + _ = prefixXOR(tc.mask) + } + }) + } +} + +// BenchmarkPrefixXORThroughput measures throughput with sequential masks +func BenchmarkPrefixXORThroughput(b *testing.B) { + // Pre-generate masks to avoid setup overhead + masks := make([]uint64, 1024) + state := uint64(0xDEADBEEFCAFEBABE) + for i := range masks { + state ^= state << 13 + state ^= state >> 7 + state ^= state << 17 + masks[i] = state + } + + idx := 0 + for b.Loop() { + _ = prefixXOR(masks[idx%len(masks)]) + idx++ + } +} + +// BenchmarkPrefixXORLatencyChain measures latency when each call depends on previous +func BenchmarkPrefixXORLatencyChain(b *testing.B) { + mask := uint64(0x5555555555555555) + for b.Loop() { + mask = prefixXOR(mask) + } + // Prevent compiler from optimizing away + if mask == 0 { + b.Fatal("unexpected zero") + } +} diff --git a/simd_scanner.go b/simd_scanner.go index a55723f..95515ca 100644 --- a/simd_scanner.go +++ b/simd_scanner.go @@ -28,6 +28,9 @@ var ( cachedCrCmp archsimd.Int8x64 cachedNlCmp archsimd.Int8x64 cachedSepCmp [cachedSepCmpCount]archsimd.Int8x64 + + // PCLMULQDQ cached value: all-ones for carryless multiplication + cachedAllOnes archsimd.Uint64x2 ) // SIMD processing constants. @@ -50,6 +53,13 @@ func init() { cachedQuoteCmp = cachedSepCmp['"'] cachedCrCmp = cachedSepCmp['\r'] cachedNlCmp = cachedSepCmp['\n'] + + // Pre-load all-ones value for carryless multiplication (PCLMULQDQ) + // Used in prefixXOR: mask × 0xFFFFFFFFFFFFFFFF computes prefix XOR + cachedAllOnes = archsimd.LoadUint64x2(&[2]uint64{ + 0xFFFFFFFFFFFFFFFF, + 0xFFFFFFFFFFFFFFFF, + }) } } @@ -63,6 +73,42 @@ func shouldUseSIMD(dataLen int) bool { return useAVX512 && dataLen >= simdMinThreshold } +// ============================================================================= +// Prefix XOR (Quote Region Mask Computation) +// ============================================================================= + +// prefixXOR computes the prefix XOR of the input mask. +// For each bit position i, the result bit is the XOR of bits 0 through i. +// +// This is used to convert a quote position mask into an "inside quotes" mask: +// +// input: 0b01001010 (quote positions at 1, 3, 6) +// output: 0b11000110 (inside quote regions) +// +// When AVX-512 is available, uses PCLMULQDQ instruction for ~3x speedup. +// Mathematical basis (carryless multiplication in GF(2)): +// +// mask × all_ones = mask × (1 + 2 + 4 + ... + 2^63) +// = mask ^ (mask << 1) ^ (mask << 2) ^ ... ^ (mask << 63) +// +// The lower 64 bits of this product give us the prefix XOR. +func prefixXOR(mask uint64) uint64 { + if useAVX512 { + // PCLMULQDQ path: ~3-4 instructions + maskVec := archsimd.LoadUint64x2(&[2]uint64{mask, 0}) + result := maskVec.CarrylessMultiply(0, 0, cachedAllOnes) + return result.GetElem(0) + } + // Scalar path: 6 shifts + 6 XORs = 12 instructions + mask ^= mask << 1 + mask ^= mask << 2 + mask ^= mask << 4 + mask ^= mask << 8 + mask ^= mask << 16 + mask ^= mask << 32 + return mask +} + // ============================================================================= // Core Data Structures // ============================================================================= @@ -391,13 +437,8 @@ func processQuotesAndSeparators(quoteMask, sepMask, nextQuoteMask uint64, state state.quoted = quoted // Invalidate separators using prefix XOR on cleaned quote mask - inQuote := quoteMaskOut - inQuote ^= inQuote << 1 - inQuote ^= inQuote << 2 - inQuote ^= inQuote << 4 - inQuote ^= inQuote << 8 - inQuote ^= inQuote << 16 - inQuote ^= inQuote << 32 + // Uses PCLMULQDQ when available for ~3x fewer instructions + inQuote := prefixXOR(quoteMaskOut) if initialQuoted != 0 { inQuote = ^inQuote @@ -410,13 +451,8 @@ func processQuotesAndSeparators(quoteMask, sepMask, nextQuoteMask uint64, state // invalidateNewlinesInQuotes removes newline bits that are inside quoted regions. func invalidateNewlinesInQuotes(quoteMask, newlineMask uint64, state *scanState) uint64 { // Prefix XOR: inQuote[i] = 1 iff positions 0..i have odd number of quotes - inQuote := quoteMask - inQuote ^= inQuote << 1 - inQuote ^= inQuote << 2 - inQuote ^= inQuote << 4 - inQuote ^= inQuote << 8 - inQuote ^= inQuote << 16 - inQuote ^= inQuote << 32 + // Uses PCLMULQDQ when available for ~3x fewer instructions + inQuote := prefixXOR(quoteMask) // If we started inside a quoted region, invert the mask if state.quoted != 0 { diff --git a/simd_scanner_test.go b/simd_scanner_test.go index 185fd0a..e983273 100644 --- a/simd_scanner_test.go +++ b/simd_scanner_test.go @@ -1160,3 +1160,165 @@ func popcount(x uint64) int { } return count } + +// ============================================================================ +// TestPrefixXOR - Test prefix XOR computation (PCLMULQDQ optimization) +// ============================================================================ + +func TestPrefixXOR(t *testing.T) { + // prefixXOR computes the cumulative XOR: result[i] = XOR of bits 0..i + // This is used for quote region detection: bit i is set if there's an + // odd number of quotes at positions 0 through i (inclusive). + tests := []struct { + name string + input uint64 + want uint64 + }{ + { + name: "empty", + input: 0, + want: 0, // no quotes = no regions + }, + { + name: "single_bit_0", + input: 1, // quote at position 0 + // All positions 0..63 have odd count (1) → all bits set + want: 0xFFFFFFFFFFFFFFFF, + }, + { + name: "single_bit_1", + input: 2, // quote at position 1 (0b10) + // pos 0: 0 quotes → 0; pos 1+: 1 quote → 1 + want: 0xFFFFFFFFFFFFFFFE, + }, + { + name: "single_bit_2", + input: 4, // quote at position 2 (0b100) + // pos 0,1: 0 quotes → 0; pos 2+: 1 quote → 1 + want: 0xFFFFFFFFFFFFFFFC, + }, + { + name: "two_adjacent_bits", + input: 0b11, // quotes at positions 0,1 + // pos 0: 1 quote → 1; pos 1+: 2 quotes → 0 + want: 0x0000000000000001, + }, + { + name: "alternating_bits_8bit", + input: 0xAA, // 0b10101010 = quotes at positions 1,3,5,7 + // pos 0: 0→0, pos 1: 1→1, pos 2: 1→1, pos 3: 2→0, + // pos 4: 2→0, pos 5: 3→1, pos 6: 3→1, pos 7: 4→0 + // Low 8 bits: 0b01100110 = 0x66 + // pos 8+: 4 quotes (even) → 0 + want: 0x0000000000000066, + }, + { + name: "quote_example", + input: 0x4A, // 0b01001010 = quotes at positions 1, 3, 6 + // pos 0: 0→0, pos 1: 1→1, pos 2: 1→1, pos 3: 2→0, + // pos 4: 2→0, pos 5: 2→0, pos 6: 3→1, pos 7+: 3→1 (odd) + // Low 8 bits: 0b11000110 = 0xC6 + // Upper bits: all 1 (odd count continues) + want: 0xFFFFFFFFFFFFFFC6, + }, + { + name: "all_ones_8bit", + input: 0xFF, // quotes at all positions 0-7 + // pos 0: 1→1, pos 1: 2→0, pos 2: 3→1, pos 3: 4→0, ... + // Pattern: 10101010... = 0x55 for low 8 bits + // pos 8+: 8 quotes (even) → 0 + want: 0x0000000000000055, + }, + { + name: "high_bit_only", + input: uint64(1) << 63, // quote at position 63 only + // pos 0-62: 0 quotes → 0; pos 63: 1 quote → 1 + want: 0x8000000000000000, + }, + { + name: "bits_0_and_63", + input: 1 | (uint64(1) << 63), // quotes at positions 0 and 63 + // pos 0: 1→1, pos 1-62: 1→1 (odd), pos 63: 2→0 + want: 0x7FFFFFFFFFFFFFFF, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := prefixXOR(tt.input) + if got != tt.want { + t.Errorf("prefixXOR(0x%016x) = 0x%016x, want 0x%016x", + tt.input, got, tt.want) + } + }) + } +} + +// TestPrefixXORQuoteRegions tests the actual use case: converting quote positions to regions +func TestPrefixXORQuoteRegions(t *testing.T) { + // prefixXOR result[i] = 1 means odd number of quotes at positions 0..i (inclusive) + // In CSV parsing context: + // - Position i has inQuote[i]=1 if we're at or after an opening quote but not past a closing quote + // - Quote chars themselves are considered "in quote" for masking purposes + tests := []struct { + name string + quotePos []int // positions where quotes appear + wantInQuote []int // positions where inQuote bit should be 1 + wantOutQuote []int // positions where inQuote bit should be 0 + initialQuoted bool // if true, we start inside a quoted region + }{ + { + name: "simple_quoted_field", + quotePos: []int{0, 5}, // "hello" - quotes at 0 and 5 + // pos 0: 1 quote → 1; pos 1-4: 1 quote → 1; pos 5: 2 quotes → 0 + wantInQuote: []int{0, 1, 2, 3, 4}, + wantOutQuote: []int{5, 6, 7, 8}, + }, + { + name: "two_quoted_fields", + quotePos: []int{0, 3, 5, 8}, // "ab","cd" - pattern at 0,3,5,8 + // pos 0-2: 1 quote → 1; pos 3-4: 2 quotes → 0; pos 5-7: 3 quotes → 1; pos 8+: 4 quotes → 0 + wantInQuote: []int{0, 1, 2, 5, 6, 7}, + wantOutQuote: []int{3, 4, 8, 9, 10}, + }, + { + name: "start_inside_quote", + quotePos: []int{5}, // closing quote at 5 + // Without inversion: pos 0-4: 0 quotes → 0; pos 5+: 1 quote → 1 + // With inversion (initialQuoted=true): pos 0-4: 1; pos 5+: 0 + wantInQuote: []int{0, 1, 2, 3, 4}, + wantOutQuote: []int{5, 6, 7, 8}, + initialQuoted: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Build quote mask + var quoteMask uint64 + for _, pos := range tt.quotePos { + quoteMask |= uint64(1) << pos + } + + // Compute in-quote mask using prefixXOR + inQuote := prefixXOR(quoteMask) + if tt.initialQuoted { + inQuote = ^inQuote + } + + // Verify expected in-quote positions + for _, pos := range tt.wantInQuote { + if inQuote&(uint64(1)<