fix: textdiff preserve whitespaces (#51)

Author: DoneDeal0

README.md

@@ -572,11 +572,12 @@ Compares two texts and returns a structured diff at a character, word, or senten
   previousText: string | null | undefined,
   currentText: string | null | undefined,
   options?: {
-    separation?: "character" | "word" | "sentence", // "word" by default
+    separation?: "character" | "word" | "sentence" // "word" by default
     accuracy?: "normal" | "high", // "normal" by default
     detectMoves?: boolean // false by default
     ignoreCase?: boolean, // false by default
     ignorePunctuation?: boolean, // false by default
+    preserveWhitespace?: boolean, // false by default
     locale?: Intl.Locale | string // undefined by default
   }
 ```
@@ -592,6 +593,7 @@ Compares two texts and returns a structured diff at a character, word, or senten
     - `true`: semantically precise, but noisier — a single insertion shifts all following tokens, breaking equality.
   - `ignoreCase`: if `true`, `hello` and `HELLO` are considered equal.
   - `ignorePunctuation`: if `true`, `hello!` and `hello` are considered equal.
+  - `preserveWhitespace`: if `true`, each token keeps the whitespace preceding it, so the original text can be rebuilt from the diff and whitespace-only edits are detected. Only available in normal accuracy mode.
   - `locale`: the locale of your text. Enables locale‑aware segmentation in high accuracy mode.
 
 **Output**

benchmark/texts.ts

@@ -50,7 +50,7 @@ export function runTextBench10KSentences() {
 
   const diff = bench("diff", 1, () => diffSentences(prev, curr, {}));
   const superdiff = bench("Superdiff", 1, () => {
-    getTextDiff(prev, curr, { separation: "sentences" });
+    getTextDiff(prev, curr, { separation: "sentence" });
   });
   return { superdiff, diff };
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@donedeal0/superdiff",
-  "version": "4.2.3",
+  "version": "4.2.4",
   "type": "module",
   "description": "Superdiff provides a rich and readable diff for arrays, objects, texts and coordinates. It supports stream and file inputs for handling large datasets efficiently, is battle-tested, has zero dependencies, and offers a top-tier performance.",
   "main": "dist/index.js",
@@ -11,7 +11,16 @@
     "dist"
   ],
   "exports": {
-    ".": "./dist/index.js",
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
     "./client": "./dist/client.js",
     "./server": "./dist/server.cjs"
   },

src/lib/text-diff/lcs/myers.ts

@@ -5,11 +5,15 @@ type MyersEdit =
   | { status: TextStatus.ADDED; curr: number }
   | { status: TextStatus.DELETED; prev: number };
 
-function backtrack(
-  trace: Map<number, number>[],
-  a: TextToken[],
-  b: TextToken[],
-): MyersEdit[] {
+type Trace = Int32Array[];
+
+function readDiagonal(trace: Int32Array, k: number, d: number): number {
+  const index = k + (d + 1);
+  if (index < 0 || index >= trace.length) return 0;
+  return trace[index];
+}
+
+function backtrack(trace: Trace, a: TextToken[], b: TextToken[]): MyersEdit[] {
   let x = a.length;
   let y = b.length;
   const edits: MyersEdit[] = [];
@@ -19,13 +23,16 @@ function backtrack(
     const k = x - y;
 
     let prevK: number;
-    if (k === -d || (k !== d && (v.get(k - 1) ?? 0) < (v.get(k + 1) ?? 0))) {
+    if (
+      k === -d ||
+      (k !== d && readDiagonal(v, k - 1, d) < readDiagonal(v, k + 1, d))
+    ) {
       prevK = k + 1;
     } else {
       prevK = k - 1;
     }
 
-    const prevX = v.get(prevK) ?? 0;
+    const prevX = readDiagonal(v, prevK, d);
     const prevY = prevX - prevK;
 
     while (x > prevX && y > prevY) {
@@ -63,20 +70,20 @@ export function myersDiff(a: TextToken[], b: TextToken[]): MyersEdit[] {
   const M = b.length;
   const max = N + M;
 
-  const trace: Map<number, number>[] = [];
-  const v = new Map<number, number>();
-  v.set(1, 0);
+  const trace: Trace = [];
+  const offset = max + 1;
+  const v = new Int32Array(2 * max + 3);
 
   for (let d = 0; d <= max; d++) {
-    const vSnapshot = new Map(v);
+    trace.push(v.slice(offset - d - 1, offset + d + 2));
 
     for (let k = -d; k <= d; k += 2) {
       let x: number;
 
-      if (k === -d || (k !== d && (v.get(k - 1) ?? 0) < (v.get(k + 1) ?? 0))) {
-        x = v.get(k + 1) ?? 0;
+      if (k === -d || (k !== d && v[offset + k - 1] < v[offset + k + 1])) {
+        x = v[offset + k + 1];
       } else {
-        x = (v.get(k - 1) ?? 0) + 1;
+        x = v[offset + k - 1] + 1;
       }
 
       let y = x - k;
@@ -86,15 +93,12 @@ export function myersDiff(a: TextToken[], b: TextToken[]): MyersEdit[] {
         y++;
       }
 
-      v.set(k, x);
+      v[offset + k] = x;
 
       if (x >= N && y >= M) {
-        trace.push(vSnapshot);
         return backtrack(trace, a, b);
       }
     }
-
-    trace.push(vSnapshot);
   }
 
   return [];

src/lib/text-diff/text-diff.test.ts

@@ -5565,3 +5565,221 @@ describe("getTextDiff – with moves detection", () => {
     ).toStrictEqual(resultFrench);
   });
 });
+
+describe("getTextDiff - preserveWhitespace", () => {
+  const CODE_LINES: [string, string][] = [
+    ["    const foo = bar(1);", "    const foo = baz(2);"],
+    ["var myVar = 2;", "var myVariable = 3;"],
+    ["a  b\tc", "a  b\tc d"],
+    ["\t\tif (a && b) {  ", "\t\tif (a || b) {  "],
+    ["  return x;", "      return x;"],
+    ["\tif (a) {", "    if (a) {"],
+    ["   ", "      "],
+    ["", "  hello  "],
+    ["  hello  ", ""],
+    ["  hello  ", "  hello  "],
+  ];
+
+  const rebuildBothTexts = (diff: ReturnType<typeof getTextDiff>) => {
+    let previous = "";
+    let current = "";
+    for (const part of diff.diff) {
+      if (part.status === "updated") {
+        previous += part.previousValue ?? "";
+        current += part.value;
+        continue;
+      }
+      if (part.status !== "added") previous += part.value;
+      if (part.status !== "deleted") current += part.value;
+    }
+    return { previous, current };
+  };
+
+  it("keeps the indentation on the surrounding tokens", () => {
+    expect(
+      getTextDiff("  const a = 1;", "  const b = 1;", {
+        separation: "word",
+        preserveWhitespace: true,
+      }),
+    ).toStrictEqual({
+      type: "text",
+      status: "updated",
+      diff: [
+        { value: "  const", index: 0, previousIndex: 0, status: "equal" },
+        { value: " a", index: null, previousIndex: 1, status: "deleted" },
+        { value: " b", index: 1, previousIndex: null, status: "added" },
+        { value: " =", index: 2, previousIndex: 2, status: "equal" },
+        { value: " 1;", index: 3, previousIndex: 3, status: "equal" },
+      ],
+    });
+  });
+
+  it("reports a re-indentation that is invisible by default", () => {
+    expect(
+      getTextDiff("  return x;", "      return x;", { separation: "word" }),
+    ).toStrictEqual({
+      type: "text",
+      status: "equal",
+      diff: [
+        { value: "return", index: 0, previousIndex: 0, status: "equal" },
+        { value: "x;", index: 1, previousIndex: 1, status: "equal" },
+      ],
+    });
+
+    expect(
+      getTextDiff("  return x;", "      return x;", {
+        separation: "word",
+        preserveWhitespace: true,
+      }),
+    ).toStrictEqual({
+      type: "text",
+      status: "updated",
+      diff: [
+        { value: "  return", index: null, previousIndex: 0, status: "deleted" },
+        {
+          value: "      return",
+          index: 0,
+          previousIndex: null,
+          status: "added",
+        },
+        { value: " x;", index: 1, previousIndex: 1, status: "equal" },
+      ],
+    });
+  });
+
+  it("reports tabs converted to spaces", () => {
+    expect(
+      getTextDiff("\tif (a) {", "    if (a) {", {
+        separation: "word",
+        preserveWhitespace: true,
+      }).status,
+    ).toBe("updated");
+  });
+
+  it("stays equal when the spacing is untouched", () => {
+    expect(
+      getTextDiff("  a  b  ", "  a  b  ", {
+        separation: "word",
+        preserveWhitespace: true,
+      }).status,
+    ).toBe("equal");
+  });
+
+  it.each(CODE_LINES)("rebuilds %j and %j - word", (previous, current) => {
+    expect(
+      rebuildBothTexts(
+        getTextDiff(previous, current, {
+          separation: "word",
+          preserveWhitespace: true,
+        }),
+      ),
+    ).toEqual({ previous, current });
+  });
+
+  it.each(CODE_LINES)("rebuilds %j and %j - character", (previous, current) => {
+    expect(
+      rebuildBothTexts(
+        getTextDiff(previous, current, {
+          separation: "character",
+          preserveWhitespace: true,
+        }),
+      ),
+    ).toEqual({ previous, current });
+  });
+
+  it.each(CODE_LINES)(
+    "rebuilds %j and %j - detectMoves",
+    (previous, current) => {
+      expect(
+        rebuildBothTexts(
+          getTextDiff(previous, current, {
+            separation: "word",
+            preserveWhitespace: true,
+            detectMoves: true,
+          }),
+        ),
+      ).toEqual({ previous, current });
+    },
+  );
+
+  it("rebuilds sentences", () => {
+    const previous = "  Hello world.   How are you?  ";
+    const current = "  Hello there.   How are you?  ";
+    expect(
+      rebuildBothTexts(
+        getTextDiff(previous, current, {
+          separation: "sentence",
+          preserveWhitespace: true,
+        }),
+      ),
+    ).toEqual({ previous, current });
+  });
+
+  it("keeps surrogate pairs intact", () => {
+    const previous = "  a😀b";
+    const current = "  a😀c";
+    const diff = getTextDiff(previous, current, {
+      separation: "character",
+      preserveWhitespace: true,
+    });
+
+    expect(rebuildBothTexts(diff)).toEqual({ previous, current });
+    expect(diff.diff.some((part) => part.value === "😀")).toBe(true);
+  });
+
+  it("reports the same statuses as the default when the spacing is unchanged", () => {
+    const previous = "  const foo = bar(1);";
+    const current = "  const foo = baz(2);";
+    const statuses = (diff: ReturnType<typeof getTextDiff>) =>
+      diff.diff.map((part) => part.status);
+
+    expect(
+      statuses(
+        getTextDiff(previous, current, {
+          separation: "word",
+          preserveWhitespace: true,
+        }),
+      ),
+    ).toEqual(statuses(getTextDiff(previous, current, { separation: "word" })));
+  });
+
+  it("still honours ignoreCase", () => {
+    expect(
+      getTextDiff("  Hello", "  hello", {
+        separation: "word",
+        preserveWhitespace: true,
+        ignoreCase: true,
+      }).status,
+    ).toBe("equal");
+  });
+});
+
+describe("getTextDiff - large dissimilar inputs", () => {
+  it("diffs two large, mostly dissimilar texts without exhausting memory", () => {
+    const previous = Array.from(
+      { length: 900 },
+      (_unused, i) => `alpha_${i}(x${i}).beta;`,
+    ).join(" ");
+    const current = Array.from(
+      { length: 900 },
+      (_unused, i) => `gamma${i % 7}[y${i}] = delta_${i} + 1;`,
+    ).join(" ");
+
+    const diff = getTextDiff(previous, current, {
+      separation: "word",
+      preserveWhitespace: true,
+    });
+
+    expect(diff.status).toBe("updated");
+
+    let rebuiltPrevious = "";
+    let rebuiltCurrent = "";
+    for (const part of diff.diff) {
+      if (part.status !== "added") rebuiltPrevious += part.value;
+      if (part.status !== "deleted") rebuiltCurrent += part.value;
+    }
+
+    expect(rebuiltPrevious).toBe(previous);
+    expect(rebuiltCurrent).toBe(current);
+  }, 15_000);
+});