feat(inspector): choose the CSV escape character (doubled quote or backslash) (#1944)

* feat(inspector): choose the CSV escape character (doubled quote or backslash)

* fix(inspector): make the CSV doubled-quote escape track the selected quote character

Author: datlechin

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - In the CSV editor, split a column into several by a delimiter or regular expression, and merge a column with the one next to it using a separator. Both are single undo steps. (#1469)
 - In the CSV editor, switch the first row between header and data with `Cmd+Shift+H` when auto-detection guesses wrong. Files without a header are no longer saved with a generated header row. (#1469)
 - In the CSV editor, set the delimiter, quote character, encoding, and line ending by hand from Edit > Set CSV Properties…, then Reload to re-read the file with those settings. (#1469)
+- In the CSV editor, choose the escape character (a doubled quote or a backslash) in Set CSV Properties…, so files that escape quotes with a backslash read and save correctly. (#1469)
 - Add llama.cpp and MLX as local AI providers. Each preset points at the server's default local endpoint and needs no API key, alongside the existing Ollama and custom OpenAI-compatible options. (#1777)
 
 ### Fixed

Plugins/CSVInspectorPlugin/CSVWriter.swift

@@ -72,9 +72,8 @@ struct CSVWriter {
     func encodeRow(_ cells: [String]) -> String {
         let delimiterScalar = UnicodeScalar(dialect.delimiter)
         let quoteScalar = UnicodeScalar(dialect.quoteChar)
+        let escapeScalar = UnicodeScalar(dialect.escapeChar)
         let delimiter = String(delimiterScalar)
-        let quote = String(quoteScalar)
-        let doubledQuote = quote + quote
 
         var line = ""
         for (index, field) in cells.enumerated() {
@@ -85,8 +84,7 @@ struct CSVWriter {
                 field,
                 delimiterScalar: delimiterScalar,
                 quoteScalar: quoteScalar,
-                quote: quote,
-                doubledQuote: doubledQuote
+                escapeScalar: escapeScalar
             )
         }
         return line
@@ -108,13 +106,19 @@ struct CSVWriter {
         _ field: String,
         delimiterScalar: UnicodeScalar,
         quoteScalar: UnicodeScalar,
-        quote: String,
-        doubledQuote: String
+        escapeScalar: UnicodeScalar
     ) -> String {
         let needsQuoting = field.unicodeScalars.contains { scalar in
             scalar == delimiterScalar || scalar == quoteScalar || scalar == "\n" || scalar == "\r"
         }
         guard needsQuoting else { return field }
-        return quote + field.replacingOccurrences(of: quote, with: doubledQuote) + quote
+        let quote = String(quoteScalar)
+        if escapeScalar == quoteScalar {
+            return quote + field.replacingOccurrences(of: quote, with: quote + quote) + quote
+        }
+        let escape = String(escapeScalar)
+        var body = field.replacingOccurrences(of: escape, with: escape + escape)
+        body = body.replacingOccurrences(of: quote, with: escape + quote)
+        return quote + body + quote
     }
 }

Plugins/TableProPluginKit/CSVDialect.swift

@@ -17,6 +17,7 @@ public struct CSVDialect: Equatable, Sendable {
 
     public var delimiter: UInt8
     public var quoteChar: UInt8
+    public var escapeChar: UInt8
     public var encoding: String.Encoding
     public var lineEnding: LineEnding
     public var hasBom: Bool
@@ -30,6 +31,7 @@ public struct CSVDialect: Equatable, Sendable {
     ) {
         self.delimiter = delimiter
         self.quoteChar = quoteChar
+        self.escapeChar = quoteChar
         self.encoding = encoding
         self.lineEnding = lineEnding
         self.hasBom = hasBom

Plugins/TableProPluginKit/CSVStreamingParser.swift

@@ -10,6 +10,7 @@ public struct CSVStreamingParser: Sendable {
     public func indexRows(_ bytes: UnsafeBufferPointer<UInt8>) -> [Range<Int>] {
         var ranges: [Range<Int>] = []
         let quote = dialect.quoteChar
+        let escape = dialect.escapeChar
         let delimiter = dialect.delimiter
         let count = bytes.count
         var i = bomSkip(in: bytes)
@@ -20,8 +21,12 @@ public struct CSVStreamingParser: Sendable {
         while i < count {
             let byte = bytes[i]
             if insideQuotes {
+                if escape != quote, byte == escape, i + 1 < count {
+                    i += 2
+                    continue
+                }
                 if byte == quote {
-                    if i + 1 < count, bytes[i + 1] == quote {
+                    if escape == quote, i + 1 < count, bytes[i + 1] == quote {
                         i += 2
                         continue
                     }
@@ -69,6 +74,7 @@ public struct CSVStreamingParser: Sendable {
         var fields: [String] = []
         var field: [UInt8] = []
         let quote = dialect.quoteChar
+        let escape = dialect.escapeChar
         let delimiter = dialect.delimiter
         var insideQuotes = false
         var i = range.lowerBound
@@ -77,8 +83,13 @@ public struct CSVStreamingParser: Sendable {
         while i < end {
             let byte = bytes[i]
             if insideQuotes {
+                if escape != quote, byte == escape, i + 1 < end {
+                    field.append(bytes[i + 1])
+                    i += 2
+                    continue
+                }
                 if byte == quote {
-                    if i + 1 < end, bytes[i + 1] == quote {
+                    if escape == quote, i + 1 < end, bytes[i + 1] == quote {
                         field.append(quote)
                         i += 2
                         continue
@@ -115,6 +126,7 @@ public struct CSVStreamingParser: Sendable {
     public func field(_ bytes: UnsafeBufferPointer<UInt8>, range: Range<Int>, column: Int) -> String {
         guard column >= 0 else { return "" }
         let quote = dialect.quoteChar
+        let escape = dialect.escapeChar
         let delimiter = dialect.delimiter
         var insideQuotes = false
         var i = range.lowerBound
@@ -126,8 +138,13 @@ public struct CSVStreamingParser: Sendable {
         while i < end {
             let byte = bytes[i]
             if insideQuotes {
+                if escape != quote, byte == escape, i + 1 < end {
+                    if currentColumn == column { field.append(bytes[i + 1]) }
+                    i += 2
+                    continue
+                }
                 if byte == quote {
-                    if i + 1 < end, bytes[i + 1] == quote {
+                    if escape == quote, i + 1 < end, bytes[i + 1] == quote {
                         if currentColumn == column { field.append(quote) }
                         i += 2
                         continue

TablePro/Views/Inspector/CSVPropertiesSheet.swift

@@ -13,6 +13,7 @@ struct CSVPropertiesSheet: View {
 
     @State private var delimiterIndex: Int
     @State private var quoteIndex: Int
+    @State private var escapeIndex: Int
     @State private var encodingIndex: Int
     @State private var lineEndingIndex: Int
 
@@ -26,6 +27,7 @@ struct CSVPropertiesSheet: View {
         self.onCancel = onCancel
         _delimiterIndex = State(initialValue: CSVPropertyOptions.delimiterIndex(for: dialect.delimiter))
         _quoteIndex = State(initialValue: CSVPropertyOptions.quoteIndex(for: dialect.quoteChar))
+        _escapeIndex = State(initialValue: CSVPropertyOptions.escapeIndex(for: dialect.escapeChar))
         _encodingIndex = State(initialValue: CSVPropertyOptions.encodingIndex(for: dialect.encoding))
         _lineEndingIndex = State(initialValue: CSVPropertyOptions.lineEndingIndex(for: dialect.lineEnding))
     }
@@ -48,6 +50,11 @@ struct CSVPropertiesSheet: View {
                         Text(CSVPropertyOptions.quotes[index].label).tag(index)
                     }
                 }
+                Picker("Escape character", selection: $escapeIndex) {
+                    ForEach(CSVPropertyOptions.escapes.indices, id: \.self) { index in
+                        Text(CSVPropertyOptions.escapes[index].label).tag(index)
+                    }
+                }
                 Picker("Encoding", selection: $encodingIndex) {
                     ForEach(CSVPropertyOptions.encodings.indices, id: \.self) { index in
                         Text(CSVPropertyOptions.encodings[index].label).tag(index)
@@ -77,6 +84,7 @@ struct CSVPropertiesSheet: View {
             base: baseDialect,
             delimiterIndex: delimiterIndex,
             quoteIndex: quoteIndex,
+            escapeIndex: escapeIndex,
             encodingIndex: encodingIndex,
             lineEndingIndex: lineEndingIndex
         )

TablePro/Views/Inspector/CSVPropertyOptions.swift

@@ -21,6 +21,13 @@ enum CSVPropertyOptions {
         (String(localized: "Single Quote  '"), 0x27),
     ]
 
+    static let escapes: [(label: String, byte: UInt8)] = [
+        (String(localized: "Doubled Quote"), 0x22),
+        (String(localized: "Backslash  \\"), 0x5C),
+    ]
+
+    static let backslashEscapeIndex = 1
+
     static let encodings: [(label: String, encoding: String.Encoding)] = [
         ("UTF-8", .utf8),
         ("UTF-16 LE", .utf16LittleEndian),
@@ -43,6 +50,10 @@ enum CSVPropertyOptions {
         quotes.firstIndex { $0.byte == byte } ?? 0
     }
 
+    static func escapeIndex(for byte: UInt8) -> Int {
+        byte == escapes[backslashEscapeIndex].byte ? backslashEscapeIndex : 0
+    }
+
     static func encodingIndex(for encoding: String.Encoding) -> Int {
         encodings.firstIndex { $0.encoding == encoding } ?? 0
     }
@@ -55,15 +66,20 @@ enum CSVPropertyOptions {
         base: CSVDialect,
         delimiterIndex: Int,
         quoteIndex: Int,
+        escapeIndex: Int,
         encodingIndex: Int,
         lineEndingIndex: Int
     ) -> CSVDialect {
-        CSVDialect(
+        var dialect = CSVDialect(
             delimiter: delimiters.indices.contains(delimiterIndex) ? delimiters[delimiterIndex].byte : base.delimiter,
             quoteChar: quotes.indices.contains(quoteIndex) ? quotes[quoteIndex].byte : base.quoteChar,
             encoding: encodings.indices.contains(encodingIndex) ? encodings[encodingIndex].encoding : base.encoding,
             lineEnding: lineEndings.indices.contains(lineEndingIndex) ? lineEndings[lineEndingIndex].value : base.lineEnding,
             hasBom: base.hasBom
         )
+        dialect.escapeChar = escapeIndex == backslashEscapeIndex
+            ? escapes[backslashEscapeIndex].byte
+            : dialect.quoteChar
+        return dialect
     }
 }

TableProTests/Plugins/CSVInspectorTests.swift

@@ -77,6 +77,18 @@ struct CSVDialectDetectionTests {
         let dialect = CSVDialect.detect(from: data)
         #expect(dialect.encoding == .windowsCP1252)
     }
+
+    @Test("Default escape character is the quote character")
+    func defaultEscapeChar() {
+        #expect(CSVDialect.csv.escapeChar == 0x22)
+        let detected = CSVDialect.detect(from: "a,b\n1,2\n".data(using: .utf8)!)
+        #expect(detected.escapeChar == 0x22)
+    }
+
+    @Test("Escape character follows a non-default quote character")
+    func escapeFollowsQuote() {
+        #expect(CSVDialect(delimiter: 0x2C, quoteChar: 0x27).escapeChar == 0x27)
+    }
 }
 
 @Suite("CSVStreamingParser")
@@ -133,6 +145,37 @@ struct CSVStreamingParserTests {
         #expect(fields == ["a", #"say "hi""#, "c"])
     }
 
+    @Test("Backslash escape decodes an escaped quote to a single quote")
+    func backslashEscapesQuote() {
+        var dialect = CSVDialect.csv
+        dialect.escapeChar = 0x5C
+        let (data, ranges, parser) = parse(#""a\"b",c"# + "\n", dialect: dialect)
+        #expect(ranges.count == 1)
+        let fields = row(data, parser, ranges[0])
+        #expect(fields == [#"a"b"#, "c"])
+    }
+
+    @Test("Backslash escape decodes a doubled backslash to a single backslash")
+    func backslashEscapesBackslash() {
+        var dialect = CSVDialect.csv
+        dialect.escapeChar = 0x5C
+        let (data, ranges, parser) = parse(#""a\\b""# + "\n", dialect: dialect)
+        let fields = row(data, parser, ranges[0])
+        #expect(fields == [#"a\b"#])
+    }
+
+    @Test("field(at:column:) honors the backslash escape inside a quoted field")
+    func fieldBackslashEscape() {
+        var dialect = CSVDialect.csv
+        dialect.escapeChar = 0x5C
+        let (data, ranges, parser) = parse(#""a\"b",c"# + "\n", dialect: dialect)
+        let first = data.withUnsafeBytes { raw -> String in
+            guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return "" }
+            return parser.field(UnsafeBufferPointer(start: base, count: raw.count), range: ranges[0], column: 0)
+        }
+        #expect(first == #"a"b"#)
+    }
+
     @Test("Empty fields preserved")
     func emptyFields() {
         let (data, ranges, parser) = parse(",,,\n")
@@ -473,6 +516,20 @@ struct CSVWriterRoundTripTests {
         #expect(written.prefix(3) == Data([0xEF, 0xBB, 0xBF]))
     }
 
+    @Test("Writer doubles an embedded quote by default")
+    func writerDefaultDoublesQuote() {
+        let writer = CSVWriter(dialect: .csv)
+        #expect(writer.encodeRow([#"a"b"#, "c"]) == #""a""b",c"#)
+    }
+
+    @Test("Writer escapes an embedded quote with the escape character")
+    func writerBackslashEscapesQuote() {
+        var dialect = CSVDialect.csv
+        dialect.escapeChar = 0x5C
+        let writer = CSVWriter(dialect: dialect)
+        #expect(writer.encodeRow([#"a"b"#, "c"]) == #""a\"b",c"#)
+    }
+
     @Test("A headerless file is written without a synthetic header row")
     func roundTripHeaderless() throws {
         let source = "1,2\n3,4\n"

TableProTests/Views/CSVPropertyOptionsTests.swift

@@ -22,20 +22,36 @@ struct CSVPropertyOptionsTests {
         #expect(CSVPropertyOptions.delimiterIndex(for: 0x5E) == 0)
     }
 
-    @Test("Building a dialect from indices sets the four properties and keeps the BOM")
+    @Test("Building a dialect from indices sets every property and keeps the BOM")
     func dialectRoundTrip() {
         let base = CSVDialect(delimiter: 0x2C, hasBom: true)
         let dialect = CSVPropertyOptions.dialect(
             base: base,
             delimiterIndex: CSVPropertyOptions.delimiterIndex(for: 0x09),
             quoteIndex: CSVPropertyOptions.quoteIndex(for: 0x27),
+            escapeIndex: CSVPropertyOptions.escapeIndex(for: 0x5C),
             encodingIndex: CSVPropertyOptions.encodingIndex(for: .utf16LittleEndian),
             lineEndingIndex: CSVPropertyOptions.lineEndingIndex(for: .cr)
         )
         #expect(dialect.delimiter == 0x09)
         #expect(dialect.quoteChar == 0x27)
+        #expect(dialect.escapeChar == 0x5C)
         #expect(dialect.encoding == .utf16LittleEndian)
         #expect(dialect.lineEnding == .cr)
         #expect(dialect.hasBom)
     }
+
+    @Test("The doubled-quote escape follows the selected quote character")
+    func doubledQuoteEscapeTracksQuote() {
+        let dialect = CSVPropertyOptions.dialect(
+            base: CSVDialect(delimiter: 0x2C),
+            delimiterIndex: 0,
+            quoteIndex: CSVPropertyOptions.quoteIndex(for: 0x27),
+            escapeIndex: 0,
+            encodingIndex: 0,
+            lineEndingIndex: 0
+        )
+        #expect(dialect.quoteChar == 0x27)
+        #expect(dialect.escapeChar == 0x27)
+    }
 }

docs/features/csv-inspector.mdx

@@ -26,7 +26,7 @@ When a file opens, TablePro detects:
 - **Line ending**: CRLF, LF, or CR, whichever appears first in the first 64 KB.
 - **Header row**: row one is treated as headers when at least half of its cells are non-empty and not numbers. Otherwise TablePro generates `Column 1`, `Column 2`, ... and treats every row as data.
 
-When a guess is wrong, choose **Edit > Set CSV Properties…** to pick the delimiter, quote character, encoding, and line ending by hand, then **Reload** to re-read the file with those settings. Reload discards unsaved edits and asks first if you have any.
+When a guess is wrong, choose **Edit > Set CSV Properties…** to pick the delimiter, quote character, escape character, encoding, and line ending by hand, then **Reload** to re-read the file with those settings. The escape character is either a doubled quote (the RFC 4180 default) or a backslash. Reload discards unsaved edits and asks first if you have any.
 
 ## Pagination