perf(macronizer): sync engine build — range-chunk wordlist store, first visit 10min -> ~10s

dist/ regenerated from the engine repo (latin-macronizer-wasm): the wordlist
now persists as ~800 range-chunk records instead of 812k rows, the engine is
ready as soon as the parse finishes (persist runs in the background), and a
versioned meta record invalidates stale dictionaries.

e2e: smoke test timeout drops from 20min to 5min; added a return-visit test
that reloads the page and verifies lookups are served from the chunk store
("arma virumque cano" -> virūmque). Both pass in ~10s each.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: hellpanderrr

wiktionary_pron/CLAUDE.md

@@ -17,7 +17,7 @@ npm test           # unit + IPA engine tests (Mocha, ~3s)
 npm run test:unit  # pure JS helpers: sanitize, memoizeLocalStorage, V3/V4 lexicon decode
 npm run test:ipa   # wasmoon Lua engine: exact-IPA tests + golden files (15 languages)
 npm run test:e2e   # Playwright browser tests, excludes macronizer (~5 min: includes Russian lexicon load)
-npm run test:e2e:macronizer  # macronizer smoke test — first-run wordlist load takes ~10+ min
+npm run test:e2e:macronizer  # macronizer smoke tests (~30s; covers first-visit and return-visit wordlist paths)
 npx playwright test -g "Latin"   # run a single e2e test
 ```
 

wiktionary_pron/e2e/macronizer.spec.js

@@ -11,14 +11,14 @@ const PAGE = "/wiktionary_pron/macronizer.html";
  */
 test.describe("macronizer", () => {
   test("initializes and macronizes provinciarum", async ({ page }) => {
-    // The 812k-entry IndexedDB insert alone takes ~10 min in a fresh profile
-    // (measured ~150k entries/100s), and Playwright contexts never reuse it.
-    test.setTimeout(1_200_000);
+    // Since the range-chunk wordlist store, first visit = download + parse
+    // (~30s); the IndexedDB persist happens in the background.
+    test.setTimeout(300_000);
     await page.goto(PAGE);
 
-    // Ready when the macronize button is enabled (init + wordlist load done)
+    // Ready when the macronize button is enabled (init + wordlist parse done)
     await expect(page.locator("#macronize_btn")).toBeEnabled({
-      timeout: 1_140_000,
+      timeout: 240_000,
     });
 
     await page.fill("#text_to_macronize", "provinciarum");
@@ -34,4 +34,40 @@ test.describe("macronizer", () => {
       { timeout: 120_000 },
     );
   });
+
+  test("return visit serves the wordlist from IndexedDB chunks", async ({
+    page,
+  }) => {
+    test.setTimeout(300_000);
+    // Visit 1: parse + wait for the background chunk persist to finish
+    const persisted = page.waitForEvent("console", {
+      predicate: (m) => m.text().includes("background persist complete"),
+      timeout: 240_000,
+    });
+    await page.goto(PAGE);
+    await expect(page.locator("#macronize_btn")).toBeEnabled({
+      timeout: 240_000,
+    });
+    await persisted;
+
+    // Visit 2: same context → same IndexedDB. Must come up without re-parsing
+    // and answer lookups through the chunk store.
+    await page.reload();
+    await expect(page.locator("#macronize_btn")).toBeEnabled({
+      timeout: 60_000,
+    });
+    await page.fill("#text_to_macronize", "arma virumque cano");
+    await page.click("#macronize_btn");
+    await expect(page.locator("#resultText .ipa").first()).toHaveAttribute(
+      "content",
+      "arma",
+      { timeout: 60_000 },
+    );
+    // virum has a long u — proves chunk lookups return real entries
+    await expect(page.locator("#resultText .ipa").nth(1)).toHaveAttribute(
+      "content",
+      /vir[ūu]mque/,
+      { timeout: 60_000 },
+    );
+  });
 });

wiktionary_pron/macronizer/dist/analysis/WordlistEngine.d.ts

@@ -22,7 +22,31 @@ export declare class WordlistEngine {
     private nextSeq;
     private readonly DB_NAME;
     private readonly DB_VERSION;
-    private readonly STORE_NAME;
+    /** ~800 chunk records covering the whole wordlist, keyed by firstWord */
+    private readonly CHUNK_STORE;
+    /** Row-per-entry store for Morpheus-analyzed unknown words (small, grows
+     * incrementally — the chunk layout is immutable after load) */
+    private readonly EXTRA_STORE;
+    /** Single meta record: schema/data version + entry count */
+    private readonly META_STORE;
+    /** Bump when the packing logic changes incompatibly. */
+    private readonly SCHEMA_VERSION;
+    /** Bump when macrons.txt content changes, so returning visitors reload
+     * instead of keeping a stale dictionary forever. */
+    private readonly DATA_VERSION;
+    /** Entries per chunk. 1000 keeps a chunk ~100KB — one get() per unseen
+     * wordform neighborhood, small enough to clone cheaply. */
+    private readonly CHUNK_SIZE;
+    /** Sorted chunk keys, loaded once per session (~800 strings). */
+    private chunkKeys;
+    /** Fetched chunks by firstWord — bounded by chunk count (~800); cleared in
+     * clearEntriesCache() together with the per-word cache. */
+    private chunksCache;
+    /** Full in-memory groups map, present only in the session that parsed the
+     * file. Serves lookups instantly while chunks persist in the background. */
+    private memGroups;
+    /** Resolves when the background chunk persist finishes (tests await this). */
+    private persistPromise;
     /** Cache of Morpheus analyses by normalized wordform (for UI display) */
     private morpheusCache;
     /** In-memory cache for getAllEntries — eliminates redundant IndexedDB cursor
@@ -33,8 +57,11 @@ export declare class WordlistEngine {
      * Initialize IndexedDB database
      */
     init(): Promise<void>;
+    private idbGet;
     /**
-     * Check if database is populated
+     * Check if database is populated with the current schema+data version.
+     * A stale version (schema change or updated macrons.txt) reads as empty,
+     * which makes the caller re-download and overwrite.
      */
     isPopulated(): Promise<boolean>;
     /**
@@ -53,20 +80,33 @@ export declare class WordlistEngine {
      * Returns entries with accentedUnderscore populated
      */
     getAllEntries(wordform: string): Promise<WordlistEntry[]>;
+    /** Binary search the sorted chunk keys for the chunk that could contain
+     * `word` (greatest firstWord <= word), fetch it, and read the group. */
+    private lookupInChunks;
+    private lookupInExtras;
     /**
      * Normalize tag format (convert dots to dashes for consistency with RFTagger)
      */
     private normalizeTag;
     /**
-     * Add single entry to wordlist
+     * Add single entry (Morpheus-analyzed unknown word). Goes to the extras
+     * store — the chunk layout is immutable after the bulk load, and extras
+     * only ever exist for words the wordlist file doesn't contain.
      */
     addEntry(entry: WordlistEntry): Promise<void>;
+    /** Normalize a parsed file entry once, before grouping. */
+    private normalizeEntry;
+    /** Group entries by wordform, preserving file order within each group —
+     * the same order the old (wordform, seq) index cursor produced. */
+    private buildGroups;
     /**
-     * Batch add entries (for file loading)
+     * Batch add entries (for file loading). Packs the wordlist into ~800
+     * sorted range chunks instead of 812k individual rows — measured ~20x
+     * faster to persist, and lookups become one direct get() per chunk.
      */
     addEntries(entries: WordlistEntry[], onProgress?: (count: number) => void): Promise<void>;
     /**
-     * Clear all entries
+     * Clear all stores
      */
     clear(): Promise<void>;
     /**
@@ -87,6 +127,9 @@ export declare class WordlistEngine {
      * e.g. "a\te--------\ta\ta_"
      */
     loadFromText(text: string, onProgress?: (count: number) => void): Promise<void>;
+    /** Await the background chunk persist (no-op if none is running). Lets
+     * tests and shutdown paths ensure durability before closing the page. */
+    flush(): Promise<void>;
     /**
      * Load wordlist from URL (fetch + parse)
      */