From e83da954c86c04ef136be64328c22b8f959b8520 Mon Sep 17 00:00:00 2001 From: CUDE Date: Mon, 20 Jul 2026 15:13:37 +0200 Subject: [PATCH 1/5] fix: do not flag contractions written with a curly apostrophe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TOKEN_RE deliberately admits U+2019 so contractions tokenize whole, but that single character was enough to fail the ASCII gate on both confusable_word paths. The UTS #39 skeleton then folds it back to "'", so "I’ll" resolved to "l'll" — plain ASCII with letters — and reported as a disguised word. Every contraction typed with a smart quote ("don’t", "it’s") was a false positive, which is any text that has been through a phone keyboard or word processor. Judge a token's ASCII-ness against its straight-apostrophe form. The exemption covers punctuation only, so letters still decide: "Ivа’s" with a Cyrillic а is still caught. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/analyze.ts | 16 ++++++++++++++-- test/analyze.test.ts | 13 +++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/analyze.ts b/src/analyze.ts index a1662cf..35faec5 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -19,6 +19,18 @@ const FORMAT_RE = /\p{Cf}/u; const ASCII_PRINTABLE_RE = /^[\x20-\x7e]+$/; const ASCII_LETTER_RE = /[a-zA-Z]/; +/** + * TOKEN_RE deliberately keeps U+2019 inside words so contractions tokenize + * whole. That one character is enough to make an otherwise plain-ASCII word + * non-ASCII — and its skeleton folds straight back to "'", so "I’ll" would + * read as an ASCII word in disguise. Judge a token's ASCII-ness against its + * straight-apostrophe form so ordinary typographic punctuation is not evidence. + */ +const TYPOGRAPHIC_APOSTROPHE_RE = /’/g; +function isAsciiWord(token: string): boolean { + return ASCII_PRINTABLE_RE.test(token.replace(TYPOGRAPHIC_APOSTROPHE_RE, "'")); +} + /** * Characters adjacent to which an invisible run is legitimate: emoji and other * symbols build their sequences out of ZWJ, variation selectors and tag @@ -235,7 +247,7 @@ function analyzeToken( // OUTSIDE them — a whole word in an unexpected script is spoof evidence // on its own, no Latin context needed. let skeletonConfusable = false; - if (!mixed && letters.length >= 2 && !ASCII_PRINTABLE_RE.test(token)) { + if (!mixed && letters.length >= 2 && !isAsciiWord(token)) { const inExpectedScript = scripts.length > 0 && scripts.every((s) => expectedScripts.has(s)); const latinContext = dominantScript === 'Latin' || @@ -261,7 +273,7 @@ function analyzeToken( // context or expectedScripts. Requiring at least two styled LETTERS keeps // isolated compatibility symbols used in real text (m², №, ½, ℃, Ⅳ) out. let styledConfusable = false; - if (!mixed && letters.length >= 2 && !ASCII_PRINTABLE_RE.test(token)) { + if (!mixed && letters.length >= 2 && !isAsciiWord(token)) { const styledLetters = letters.filter((ch) => { const nf = ch.normalize('NFKC'); return nf !== ch && ASCII_PRINTABLE_RE.test(nf) && ASCII_LETTER_RE.test(nf); diff --git a/test/analyze.test.ts b/test/analyze.test.ts index 795cbea..c609ac9 100644 --- a/test/analyze.test.ts +++ b/test/analyze.test.ts @@ -108,6 +108,12 @@ describe('analyze — legitimate content', () => { ['Persian with ZWNJ', 'می‌خواهم بروم'], ['emoji ZWJ sequences', 'family 👨‍👩‍👧‍👦 emoji ✌🏻'], ['numbers and symbols', 'Call +1 555-0100 re: 20MM+ ARR'], + // U+2019 is a token character, so contractions are ASCII words carrying one + // curly apostrophe — their skeleton folds it back to "'", which must not + // read as a disguised ASCII word. + ['typographic apostrophes', 'Sounds good, I’ll call — don’t worry, it’s Kaysie’s job.'], + ['& apos; in HTML', 'Sounds good, I'll call — don't worry, it's Kaysie's job.'], + ['test', 'Hansen&sons'], ['empty string', ''], ]; @@ -119,6 +125,13 @@ describe('analyze — legitimate content', () => { expect(r.normalized).toBe(text); }); + it('still flags a lookalike word that carries a curly apostrophe', () => { + // The apostrophe exemption is about punctuation only — the letters still + // decide. "Ivаn’s" hides a Cyrillic "а". + const r = analyze('Ivа’s'); + expect(r.signals.mixed_script).toBe(true); + }); + it('does not call a styled word confusable unless the whole word folds to ASCII', () => { // "𝗉𝗋é" has two styled letters that individually fold to ASCII, but the // token folds to "pré" — still non-ASCII, so it is not a disguised word. From 4ae1dc1c4ff341079e97125da880473e80eafd19 Mon Sep 17 00:00:00 2001 From: CUDE Date: Mon, 20 Jul 2026 15:28:13 +0200 Subject: [PATCH 2/5] feat: separate decode damage from spoofing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit U+FFFD was classified as `illegal`, so a message whose name field had been mangled upstream reported as a spoofing attack. On a 115-row sample of real SMS traffic that was 70 rows — 61% of every flag — all of them one broken merge field ("Jos​​ Luis", "Jes​​s") in a plumbing company's appointment reminders. U+FFFD cannot evidence intent. A decoder emits it when handed bytes it cannot read, and those bytes are already destroyed by the time it exists, so no payload survives inside it. That is what makes it safe to exempt from `spoofed` without opening a bypass — unlike mojibake fingerprinting ("looks like broken UTF-8"), which an attacker could imitate on purpose to suppress detection, and which is deliberately NOT implemented here. The signal is reported, not hidden: `signals.encoding_damage` and the word findings still fire, so data-quality alerting keeps working. It just does not assert an attack. The normalizer also leaves U+FFFD in place — stripping it would repair a corrupted message into one that reads as intact. C1 controls, NUL and non-characters stay `illegal`: those are real bytes that survived, so they remain attacker-controllable. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/chilled-pugs-shave.md | 23 ++++++++++++++++ README.md | 29 ++++++++++++++++++-- src/analyze.ts | 47 ++++++++++++++++++++++++++++---- src/index.ts | 1 + src/types.ts | 24 ++++++++++++++-- test/readme.test.ts | 12 +++++++- test/spoof-suite.test.ts | 32 ++++++++++++++++++++-- 7 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 .changeset/chilled-pugs-shave.md diff --git a/.changeset/chilled-pugs-shave.md b/.changeset/chilled-pugs-shave.md new file mode 100644 index 0000000..fee5195 --- /dev/null +++ b/.changeset/chilled-pugs-shave.md @@ -0,0 +1,23 @@ +--- +'@moderation-api/unicode-spoofing': minor +--- + +Separate decode damage from spoofing with a new `encoding_damage` signal. + +U+FFFD used to count as `illegal`, so a message whose name field had been +mangled upstream — `Hi Jos�� Luis` — reported as a spoofing attack. Across a +115-row sample of real SMS traffic that was 61% of all flags. + +U+FFFD is a decoder's output, never an author's input: whatever the original +bytes were, they are gone by the time the character exists, so it can carry no +payload. It now reports as `encoding_damage`, which is included in `signals` +and `words` but deliberately does **not** set `spoofed`, and the normalizer +leaves it in place rather than silently repairing a corrupted message. + +`SpoofSignal` gains a member and `signals` gains a key, so an exhaustive +`switch` or a strict `toEqual` on the signals object will need updating. +Genuinely illegal code points (NUL, C1 controls, non-characters) are unchanged. + +A `SPOOFING_SIGNALS` constant is exported alongside `SPOOF_SIGNALS` — the same +list minus `encoding_damage` — so callers can partition the two the same way +`spoofed` does. diff --git a/README.md b/README.md index 9dd0b53..92098a5 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ r.signals; // invisible: false, // zalgo: false, // illegal: false, +// encoding_damage: false, // } r.words; @@ -75,13 +76,34 @@ a stray NUL byte: ```ts const r = analyze('НОТ busіnеss: fr\u200Bee cr̸͈͖͡ypto\u0000'); -r.signals; // every one of the five is true +r.signals; // every spoofing signal is true r.normalized; // 'HOT business: free crypto' r.counts; // { wordsTotal: 4, wordsAffected: 5 } skeleton('раураl') === skeleton('paypal'); // true (UTS #39 comparison) ``` +### Broken text is not an attack + +`encoding_damage` is the one signal that does **not** set `spoofed`. U+FFFD is +what a decoder emits when handed bytes it cannot read — a name mangled +somewhere upstream, not a disguise: + +```ts +const r = analyze('Hi Jos�� Luis, your appointment is confirmed.'); + +r.signals.encoding_damage; // true +r.spoofed; // false <- a broken pipeline, not an attacker +r.changed; // false <- left alone, not silently "repaired" +``` + +The distinction is deliberate, and it is safe to make: whatever those bytes +were, the decoder already destroyed them, so no payload can survive inside a +U+FFFD for an attacker to exploit. Nothing is hidden from you — the signal and +its word findings are still reported, so you can alert on data quality without +it counting as a spoofing verdict. Collapsing the two means every mojibake'd +`José` reads as an attack. + ## What it catches Every example below runs as a test — see [`test/readme.test.ts`](./test/readme.test.ts), @@ -189,7 +211,8 @@ analyze('Ship it 🎉 👨\u200D👩\u200D👧 ℹ\uFE0F').spoofed; // false — | `confusable_word` | Whole word is a Latin lookalike (UTS #39 skeleton resolves to ASCII) | `НОТ` → `HOT`, `ＨＯＴ`, `𝐇𝐎𝐓` | | `invisible` | Characters that render as nothing, in or between words | zero-width, bidi, tag chars, fillers | | `zalgo` | Combining marks stacked beyond orthographic depth (≥3 per base) | `Z̸̢̬a̛lg̕o` | -| `illegal` | Control, non-character, or replacement code points anywhere in text | `NUL`, `U+FFFE`, `U+FFFD` | +| `illegal` | Control or non-character code points anywhere in text | `NUL`, `U+0085`, `U+FFFE` | +| `encoding_damage` | Decode damage — reported, but never sets `spoofed` (see above) | `Jos��` (a mangled `José`) | What `invisible` covers: format characters (zero-width space/joiner, word joiner, …), bidi controls incl. the Trojan Source overrides, tag characters, @@ -204,7 +227,7 @@ against a name instead of a hardcoded string. | Export | What it is | | -------------------------------- | ----------------------------------------------------------------------------------------------------- | -| `SPOOF_SIGNALS` | All five signal names — the keys of `result.signals` | +| `SPOOF_SIGNALS` | All six signal names — the keys of `result.signals` | | `SCRIPT_NAMES` | Every script the classifier can name — the possible values of `dominantScript` and `words[].scripts` | | `SUPPORTED_SCRIPTS` | The subset of `SCRIPT_NAMES` this runtime's Unicode tables support (all of them, on a current engine) | | `PSEUDO_SCRIPTS` | `Common`, `Inherited`, `Unknown` — classifications that are not scripts and never appear in a finding | diff --git a/src/analyze.ts b/src/analyze.ts index 35faec5..25fc850 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -1,6 +1,12 @@ import { foldChar, skeleton } from './confusables'; import { analyzeWordScripts, primaryScript, type PseudoScript, type ScriptName } from './scripts'; -import type { AnalysisResult, AnalyzeOptions, SpoofSignal, WordFinding } from './types'; +import { + SPOOFING_SIGNALS, + type AnalysisResult, + type AnalyzeOptions, + type SpoofSignal, + type WordFinding, +} from './types'; /** * Tokens are runs of letters, marks, digits, format (invisible) characters, @@ -139,8 +145,10 @@ export const ZALGO_MARK_RUN = 3; * Code points that have no business appearing in ordinary text and that the * token scanner would otherwise silently drop (they are not letters, marks, * numbers or format characters, so they never enter a word): C0/C1 controls, - * DEL, Unicode non-characters and the replacement character. TAB, LF and CR - * are the only C0 controls treated as legitimate whitespace. + * DEL and Unicode non-characters. TAB, LF and CR are the only C0 controls + * treated as legitimate whitespace. + * + * U+FFFD is NOT here — see `isEncodingDamage`. */ function isIllegalCodePoint(cp: number): boolean { if (cp === 0x09 || cp === 0x0a || cp === 0x0d) return false; // TAB, LF, CR @@ -148,10 +156,25 @@ function isIllegalCodePoint(cp: number): boolean { if (cp >= 0x7f && cp <= 0x9f) return true; // DEL + C1 controls if (cp >= 0xfdd0 && cp <= 0xfdef) return true; // non-characters (BMP block) if ((cp & 0xfffe) === 0xfffe) return true; // U+xxFFFE / U+xxFFFF in every plane - if (cp === 0xfffd) return true; // replacement character (mojibake / decode damage) return false; } +/** + * U+FFFD REPLACEMENT CHARACTER: a decoder emitted this because bytes it was + * handed were not valid in the encoding it assumed — "José" arriving as + * "Jos��". It reports a broken pipeline upstream, never intent: + * whatever the original bytes were, they are gone by the time this character + * exists, so no payload survives inside it for an attacker to exploit. That is + * what makes it safe to report without calling the text spoofed, and it is why + * this is a separate signal rather than an exemption inside `illegal`. + * + * It is also left in place by the normalizer. Stripping it would silently + * repair a corrupted message into one that reads as intact. + */ +function isEncodingDamage(cp: number): boolean { + return cp === 0xfffd; +} + /** * The whole character ending at `i` — stepping back over a surrogate pair so an * astral neighbour (an emoji, typically) reads as itself and not as half of one. @@ -169,6 +192,7 @@ function emptySignals(): Record { invisible: false, zalgo: false, illegal: false, + encoding_damage: false, }; } @@ -445,6 +469,19 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes i += width; } + // Decode damage. A run reports as ONE finding: a single destroyed character + // usually yields several U+FFFDs (a two-byte "é" becomes two), and per-code- + // point findings would overstate how much of the text is broken. Nothing is + // replaced — see `isEncodingDamage`. + for (let i = 0; i < text.length; i += 1) { + if (!isEncodingDamage(text.charCodeAt(i))) continue; + let end = i + 1; + while (end < text.length && isEncodingDamage(text.charCodeAt(end))) end += 1; + signals.encoding_damage = true; + words.push({ word: text.slice(i, end), index: i, signals: ['encoding_damage'], scripts: [] }); + i = end - 1; + } + // Compatibility-styled letter runs the token scanner skips because the glyphs // are symbols, not letters (category So): circled "Ⓐⓓⓜⓘⓝ", parenthesized, // squared. A run of >=2 such glyphs whose NFKC form is an ASCII word is the @@ -520,7 +557,7 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes } return { - spoofed: words.length > 0, + spoofed: SPOOFING_SIGNALS.some((s) => signals[s]), signals, words, counts: { diff --git a/src/index.ts b/src/index.ts index c5525d5..903a19e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,7 @@ export { } from './scripts'; export { SPOOF_SIGNALS, + SPOOFING_SIGNALS, type AnalysisResult, type AnalyzeOptions, type SpoofSignal, diff --git a/src/types.ts b/src/types.ts index fc8cfd7..b8ef4b0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,12 @@ import type { ScriptName } from './scripts'; -export type SpoofSignal = 'mixed_script' | 'confusable_word' | 'invisible' | 'zalgo' | 'illegal'; +export type SpoofSignal = + | 'mixed_script' + | 'confusable_word' + | 'invisible' + | 'zalgo' + | 'illegal' + | 'encoding_damage'; export const SPOOF_SIGNALS: readonly SpoofSignal[] = [ 'mixed_script', @@ -8,8 +14,19 @@ export const SPOOF_SIGNALS: readonly SpoofSignal[] = [ 'invisible', 'zalgo', 'illegal', + 'encoding_damage', ]; +/** + * Signals that assert *intent* — someone disguised this text. `encoding_damage` + * is deliberately absent: U+FFFD is a decoder's output, not an author's input, + * so it evidences a broken pipeline rather than an attacker. It is reported, + * but it does not make a result `spoofed`. + */ +export const SPOOFING_SIGNALS: readonly SpoofSignal[] = SPOOF_SIGNALS.filter( + (s) => s !== 'encoding_damage', +); + export interface WordFinding { /** * The token exactly as it appears in the input. For an `illegal` finding @@ -40,7 +57,10 @@ export interface AnalyzeOptions { } export interface AnalysisResult { - /** True when any signal fired. */ + /** + * True when any *spoofing* signal fired. `encoding_damage` alone does not + * set this: broken text is a data-quality problem, not an attack. + */ spoofed: boolean; /** Which signals fired anywhere in the text. */ signals: Record; diff --git a/test/readme.test.ts b/test/readme.test.ts index 96508e6..1b712ad 100644 --- a/test/readme.test.ts +++ b/test/readme.test.ts @@ -22,6 +22,7 @@ describe('README — usage', () => { invisible: false, zalgo: false, illegal: false, + encoding_damage: false, }); expect(r.words.map((w) => ({ word: w.word, index: w.index, skeleton: w.skeleton }))).toEqual([ { word: 'Ｇｅｔ', index: 0, skeleton: 'Get' }, @@ -46,7 +47,7 @@ describe('README — usage', () => { expect(analyze('Verify your раураl account').normalized).toBe('Verify your paypal account'); }); - it('carries all five signals at once', () => { + it('carries every spoofing signal at once', () => { const r = analyze('НОТ busіnеss: fr\u200Bee cr̸͈͖͡ypto\u0000'); expect(r.signals).toEqual({ @@ -55,12 +56,21 @@ describe('README — usage', () => { invisible: true, zalgo: true, illegal: true, + encoding_damage: false, }); expect(r.normalized).toBe('HOT business: free crypto'); expect(r.counts).toEqual({ wordsTotal: 4, wordsAffected: 5 }); expect(skeleton('раураl') === skeleton('paypal')).toBe(true); }); + + it('reports decode damage without calling it spoofed', () => { + const r = analyze('Hi Jos�� Luis, your appointment is confirmed.'); + + expect(r.signals.encoding_damage).toBe(true); + expect(r.spoofed).toBe(false); + expect(r.changed).toBe(false); + }); }); describe('README — Trojan Source', () => { diff --git a/test/spoof-suite.test.ts b/test/spoof-suite.test.ts index 1212868..27d4aad 100644 --- a/test/spoof-suite.test.ts +++ b/test/spoof-suite.test.ts @@ -652,8 +652,36 @@ describe('control & non-character code points', () => { expect(sig(run('x￾y'))).toBe(true); }); - it('flags the object-replacement / replacement char (U+FFFD)', () => { - expect(sig(run('data�'))).toBe(true); + it('reports the replacement char (U+FFFD) as encoding damage, not spoofing', () => { + // Deliberate exception to this section's rule. U+FFFD is emitted by a + // decoder handed bytes it could not read, so it evidences a broken pipeline + // rather than an author's intent — and the original bytes are already gone, + // so it can carry no payload. It is still reported; it just is not spoofing. + const r = analyze('data�'); + expect(r.signals.encoding_damage).toBe(true); + expect(r.words.map((w) => w.signals).flat()).toContain('encoding_damage'); + expect(sig(r)).toBe(false); + }); + + it('leaves the replacement char in place rather than silently repairing it', () => { + const r = analyze('Jos�� Luis'); + expect(r.normalized).toBe('Jos�� Luis'); + expect(r.changed).toBe(false); + }); + + it('does not let decode damage mask a real spoof in the same message', () => { + // The dangerous direction: a corrupted message that ALSO carries a genuine + // lookalike must still be spoofed. Damage suppresses nothing but itself. + // 'раураl' is Cyrillic except its final Latin 'l', so it reports as + // mixed_script rather than confusable_word. + const r = analyze('Hi Jos��, verify your раураl account'); + expect(r.signals.encoding_damage).toBe(true); + expect(r.signals.mixed_script).toBe(true); + expect(r.spoofed).toBe(true); + // Each is its own finding — the damage does not absorb the spoof. + expect(r.words.map((w) => w.signals)).toEqual([['encoding_damage'], ['mixed_script']]); + // The spoof is still resolved; the damage is still left alone. + expect(r.normalized).toBe('Hi Jos��, verify your paypal account'); }); it('does NOT flag ordinary newlines/tabs in multi-line text', () => { From d588538f36baa54c28291448718b998fd95f3b3e Mon Sep 17 00:00:00 2001 From: CUDE Date: Mon, 20 Jul 2026 16:03:35 +0200 Subject: [PATCH 3/5] feat: treat whitespace-isolated zero-width runs as inert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a 48-row sample of real SMS traffic, 17 were one company's template with two zero-width spaces sitting between a colon and a line break, left behind by a rich-text editor. A zero-width character in that position cannot split a word — the break is already there — so it changes nothing about how the text renders or tokenizes, and reporting it read as accusing a sender of attacking their own recipients. The rule requires whitespace or a string boundary on BOTH sides, and that is the whole of its safety. My first attempt accepted a break on EITHER side and the existing suite caught it immediately: `admin` and `a = = b` stopped flagging. Those are not inert — a trailing zero-width renders as the bare word while comparing unequal to it, which passes an exact match, and the same goes for `Valencia` glued to the front of a token. Length is the second limit: every position in a run carries at least a bit, so past ZERO_WIDTH_INERT_RUN (4) a run is a payload channel wherever it sits. Scoped to characters that are zero-width AND non-reordering. Bidi controls reorder the line, tag characters carry a payload, and blank glyphs occupy space, so none become inert next to a space — Trojan Source and ASCII smuggling are untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/olive-pugs-clap.md | 22 +++++++++ src/analyze.ts | 89 +++++++++++++++++++++++++++++++++-- src/index.ts | 7 ++- test/invisible.test.ts | 28 +++++++++-- 4 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 .changeset/olive-pugs-clap.md diff --git a/.changeset/olive-pugs-clap.md b/.changeset/olive-pugs-clap.md new file mode 100644 index 0000000..d70c505 --- /dev/null +++ b/.changeset/olive-pugs-clap.md @@ -0,0 +1,22 @@ +--- +'@moderation-api/unicode-spoofing': minor +--- + +Stop reporting zero-width runs that are isolated in whitespace. + +A zero-width character evades a filter by splitting a word (`free`). +A run with whitespace (or a string boundary) on BOTH sides splits nothing — +the break it would create is already there — so it changes neither rendering +nor tokenization. Rich-text editors leave these in templates constantly, and +reporting them told senders their own newsletter was an attack. + +Requiring isolation on both sides is the safety of the rule. One-sided contact +still does work: `admin` renders as `admin` but compares unequal to it, +and `Valencia` glues to the front of a token the same way, so both are +still reported. Runs longer than the new exported `ZERO_WIDTH_INERT_RUN` (4) +are reported wherever they sit, because length alone makes a run a payload +channel regardless of position. + +Scoped to genuinely zero-width, non-reordering characters (U+200B, U+200C, +U+200D, U+2060, U+FEFF). Bidi controls, tag characters and blank glyphs are +untouched — Trojan Source and ASCII smuggling detection are unaffected. diff --git a/src/analyze.ts b/src/analyze.ts index 25fc850..911f066 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -138,6 +138,61 @@ export const FORMAT_CHAR_SCRIPTS: readonly ScriptName[] = [ const FORMAT_CHAR_SCRIPT_SET = new Set(FORMAT_CHAR_SCRIPTS); +/** + * Zero-width AND non-reordering: these render as nothing and move nothing. + * Deliberately excludes the other invisibles, which stay reportable wherever + * they sit — bidi controls reorder the line (Trojan Source), tag characters + * carry a payload (ASCII smuggling), and blank glyphs occupy space, so none of + * them are inert just because a space is next to them. + */ +const ZERO_WIDTH_CHARS = new Set([ + 0x200b, // ZERO WIDTH SPACE + 0x200c, // ZERO WIDTH NON-JOINER + 0x200d, // ZERO WIDTH JOINER + 0x2060, // WORD JOINER + 0xfeff, // ZERO WIDTH NO-BREAK SPACE +]); + +/** + * Longest run of zero-width characters still treated as inert when it is + * isolated in whitespace. Each position in a run carries at least a bit, so a + * long run encodes a payload wherever it sits and is always reported. + */ +export const ZERO_WIDTH_INERT_RUN = 4; + +/** + * Marks zero-width runs that touch no text at all: whitespace (or a string + * boundary) on BOTH sides. Such a run splits no word and joins no token — it + * sits in a gap that already exists, so it changes neither what the text + * renders as nor how it tokenizes, and reporting it is noise. + * + * Requiring both sides is the whole safety of the rule. A run touching text on + * ONE side is still doing work: "admin" renders as "admin" but compares + * unequal to it, and "Valencia" glues to the front of a token the same + * way — both evade exact matching without splitting anything. Only complete + * isolation makes a zero-width run inert. + * + * Computed over the whole text because the judgement needs both neighbours, + * which neither the per-token nor the standalone pass can see on its own. + */ +function inertZeroWidthMask(text: string): Uint8Array { + const mask = new Uint8Array(text.length); + for (let i = 0; i < text.length;) { + if (!ZERO_WIDTH_CHARS.has(text.charCodeAt(i))) { + i += 1; + continue; + } + let end = i + 1; + while (end < text.length && ZERO_WIDTH_CHARS.has(text.charCodeAt(end))) end += 1; + // A string boundary counts as whitespace: there is no text on that side. + const isolated = + (i === 0 || /\s/.test(text[i - 1]!)) && (end === text.length || /\s/.test(text[end]!)); + if (isolated && end - i <= ZERO_WIDTH_INERT_RUN) mask.fill(1, i, end); + i = end; + } + return mask; +} + /** Combining marks stacked deeper than this on one base = zalgo. */ export const ZALGO_MARK_RUN = 3; @@ -208,8 +263,21 @@ function analyzeToken( dominantScript: ScriptName | null, anyMixedInMessage: boolean, expectedScripts: Set, + inertMask: Uint8Array, + tokenStart: number, ): TokenAnalysis { const chars = [...token]; + // Aligned with `chars`, so the inert judgement — made over the whole text — + // is readable per code point here. Stepping by `ch.length` keeps astral + // characters from shifting the mapping into the UTF-16 mask. + const inertAt: boolean[] = []; + { + let offset = tokenStart; + for (const ch of chars) { + inertAt.push(inertMask[offset] === 1); + offset += ch.length; + } + } // Blank glyphs are excluded before script analysis: a Hangul filler is a // Hangul *letter* to the property tables, and counting it as one would report // "free" as a Latin/Hangul mix rather than as a hidden character. @@ -244,7 +312,8 @@ function analyzeToken( if (base === undefined || !LETTER_RE.test(base) || PICTOGRAPHIC_RE.test(base)) continue; if (!VARIATION_BASE_SCRIPTS.has(primaryScript(base))) strayVariation.add(i); } - const hasInvisible = chars.some((ch) => isInvisibleChar(ch)) || strayVariation.size > 0; + const hasInvisible = + chars.some((ch, i) => isInvisibleChar(ch) && !inertAt[i]) || strayVariation.size > 0; if (hasInvisible && !scriptExempt) signals.push('invisible'); // zalgo: combining marks stacked beyond orthographic depth. @@ -323,7 +392,10 @@ function analyzeToken( const dropInvisible = signals.includes('invisible'); normalized = chars .map((ch, i) => { - if (dropInvisible && (isInvisibleChar(ch) || strayVariation.has(i))) return ''; + // Inert characters survive: they were not evidence, so removing them + // would edit the message without cause. + if (dropInvisible && ((isInvisibleChar(ch) && !inertAt[i]) || strayVariation.has(i))) + return ''; if (zalgo && MARK_RE.test(ch)) return ''; if (nfkcFold) return ch; return foldLookalikes ? foldChar(ch) : ch; @@ -376,10 +448,18 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes const signals = emptySignals(); const words: WordFinding[] = []; const replacements: Array<{ start: number; end: number; value: string }> = []; + const inertMask = inertZeroWidthMask(text); for (const match of tokens) { const token = match[0]; - const result = analyzeToken(token, dominantScript, anyMixedInMessage, expectedScripts); + const result = analyzeToken( + token, + dominantScript, + anyMixedInMessage, + expectedScripts, + inertMask, + match.index, + ); if (result.signals.length === 0) continue; for (const s of result.signals) signals[s] = true; @@ -418,7 +498,7 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes const cp = text.codePointAt(i)!; const width = cp > 0xffff ? 2 : 1; const ch = String.fromCodePoint(cp); - if (!(isInvisibleChar(ch) || isVariationSelector(cp)) || inWord(i)) { + if (!(isInvisibleChar(ch) || isVariationSelector(cp)) || inWord(i) || inertMask[i] === 1) { i += width; continue; } @@ -429,6 +509,7 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes const next = text.codePointAt(end)!; const nextCh = String.fromCodePoint(next); if (!(isInvisibleChar(nextCh) || isVariationSelector(next)) || inWord(end)) break; + if (inertMask[end] === 1) break; end += next > 0xffff ? 2 : 1; } diff --git a/src/index.ts b/src/index.ts index 903a19e..9999aaa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,9 @@ -export { analyze, FORMAT_CHAR_SCRIPTS, ZALGO_MARK_RUN } from './analyze'; +export { + analyze, + FORMAT_CHAR_SCRIPTS, + ZALGO_MARK_RUN, + ZERO_WIDTH_INERT_RUN, +} from './analyze'; export { skeleton } from './confusables'; export { UNICODE_VERSION, DATA_DATE } from './data/confusables.generated'; export { diff --git a/test/invisible.test.ts b/test/invisible.test.ts index 91c94e1..d93f5be 100644 --- a/test/invisible.test.ts +++ b/test/invisible.test.ts @@ -15,7 +15,7 @@ */ import { describe, it, expect } from 'vitest'; -import { analyze } from '../src'; +import { analyze, ZERO_WIDTH_INERT_RUN } from '../src'; /** No unpaired surrogate survived the rewrite (String#isWellFormed needs ES2024). */ const wellFormed = (s: string) => @@ -275,10 +275,30 @@ describe('invisible characters that belong to no word', () => { expect(r.normalized).toBe('if (level != "user // admin check ") {'); }); - it('flags a lone invisible between two spaces', () => { + it('leaves a zero-width run isolated in whitespace alone', () => { + // Nothing to split: the break it would create is already there. This shape + // is what rich-text editors leave behind in templates, and reporting it + // told users their own newsletter was an attack. const r = analyze('free \u200B money'); - expect(r.spoofed).toBe(true); - expect(r.normalized).toBe('free money'); + expect(r.spoofed).toBe(false); + expect(r.normalized).toBe('free \u200B money'); + }); + + it('still flags a zero-width run that touches text on either side', () => { + // One-sided contact is not inert: these render identically to the bare + // word but compare unequal to it, which is enough to pass an exact match. + for (const text of ['admin\u200B', '\u200Badmin', 'ad\u200Bmin', 'a = \u200B= b']) { + expect(analyze(text).signals.invisible).toBe(true); + } + }); + + it('flags a zero-width run in whitespace once it is long enough to carry data', () => { + // Isolation stops excusing a run at ZERO_WIDTH_INERT_RUN: every position + // is at least a bit, so length alone makes it a payload channel. + const inert = `free ${'\u200B'.repeat(ZERO_WIDTH_INERT_RUN)} money`; + const payload = `free ${'\u200B'.repeat(ZERO_WIDTH_INERT_RUN + 1)} money`; + expect(analyze(inert).spoofed).toBe(false); + expect(analyze(payload).signals.invisible).toBe(true); }); it('flags an invisible run between punctuation', () => { From 537a8dc47e9471fb8cea8654414334fff0246bb8 Mon Sep 17 00:00:00 2001 From: CUDE Date: Mon, 20 Jul 2026 16:22:52 +0200 Subject: [PATCH 4/5] feat: judge Latin words on Identifier_Status, not skeleton luck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Ayalaæs Lashes" reported as a disguised word, and normalized to "Ayalaaes". The message was a corrupted "Ayala's" — a codepage swap had put æ where the apostrophe belonged — but nothing about it was a disguise, and our correction made it worse. The cause is that a Latin word was being skeleton-tested against Latin. There is no cross-script evidence to find there, so all the test measured was whether the fold happened to land in ASCII. It does for ordinary European letters, and it does so arbitrarily: æ is a ligature so UTS #39 dissolves it to "ae", while ø keeps its stroke as a combining mark and never gets there. Ægir and Þór were reported; Ålborg and Straße were not. Coin flip, decided by table shape. Identifier_Status is the distinction Unicode defines for exactly this question. æ ø å ß þ œ ı and the Hawaiian ʻokina are Allowed — letters of living alphabets. IPA alpha and the fi ligature are Restricted. Requiring a Restricted character keeps intra-Latin homoglyphs like "pɑypal", which mixed_script and the cross-script path are both blind to, while clearing every Nordic, Turkish, German, French and Hawaiian word we were indicting. Scoped to Latin words deliberately. Cyrillic а and о are Allowed as well — they are ordinary Russian — so a universal gate would have let раураl through. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/great-moons-judge.md | 25 ++++++ README.md | 20 ++++- package.json | 2 +- scripts/generate-identifier-status.mjs | 106 ++++++++++++++++++++++++ src/analyze.ts | 25 +++++- src/data/identifier-status.generated.ts | 13 +++ src/identifier-status.ts | 49 +++++++++++ test/analyze.test.ts | 37 +++++++++ 8 files changed, 271 insertions(+), 6 deletions(-) create mode 100644 .changeset/great-moons-judge.md create mode 100644 scripts/generate-identifier-status.mjs create mode 100644 src/data/identifier-status.generated.ts create mode 100644 src/identifier-status.ts diff --git a/.changeset/great-moons-judge.md b/.changeset/great-moons-judge.md new file mode 100644 index 0000000..1212903 --- /dev/null +++ b/.changeset/great-moons-judge.md @@ -0,0 +1,25 @@ +--- +'@moderation-api/unicode-spoofing': minor +--- + +Judge Latin words on UTS #39 Identifier_Status instead of skeleton luck. + +A word already written in Latin cannot be impersonating Latin, so the +cross-script skeleton test measured only whether its fold happened to reach +ASCII — which it does for ordinary European orthography, and arbitrarily. +`æ` is a ligature that UTS #39 dissolves to `ae`, while `ø` keeps its stroke as +a combining mark and never reaches ASCII, so `Ægir` was reported as a disguised +word and `Ålborg` was not. Same alphabet, opposite verdicts. + +A Latin word now needs at least one character Unicode marks **Restricted** in +`IdentifierStatus.txt` before its skeleton counts. `Ægir`, `Þór`, `Straße`, +`cœur`, `ısıtır` and `Hawaiʻi` are all Allowed and pass through; `pɑypal` +(U+0251 IPA ALPHA) and `first` (a compatibility ligature) are Restricted and are +still caught — intra-Latin homoglyphs that no script comparison can see. + +The gate applies ONLY to Latin words. Cyrillic `а`/`о` are Allowed too, being +ordinary Russian, so applying it everywhere would let `раураl` walk through. + +Adds `src/data/identifier-status.generated.ts` (~7 KB) and +`scripts/generate-identifier-status.mjs`; `npm run generate:data` refreshes +both tables. diff --git a/README.md b/README.md index 92098a5..f9ea427 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ you the de-obfuscated text back. Zero runtime dependencies. Script classification rides the JS engine's own Unicode property tables (`\p{Script=…}`); the only shipped data is the -UTS #39 confusables table, generated from the Unicode Consortium's -`confusables.txt` and pinned to a Unicode version. +UTS #39 security tables, generated from the Unicode Consortium's +`confusables.txt` and `IdentifierStatus.txt` and pinned to a Unicode version. Ships as dual ESM/CommonJS with bundled type declarations. Node 20+. @@ -235,6 +235,7 @@ against a name instead of a hardcoded string. | `LEGITIMATE_SCRIPT_COMBINATIONS` | Script mixes that never fire `mixed_script` (Japanese, Korean, Bopomofo-annotated Chinese) | | `ZALGO_MARK_RUN` | Combining marks per base at which `zalgo` fires | | `UNICODE_VERSION`, `DATA_DATE` | Which UTS #39 confusables table is compiled in | +| `ZERO_WIDTH_INERT_RUN` | Longest zero-width run still treated as inert when isolated in whitespace | `ScriptName` types all of these, so `expectedScripts` and any comparison against `dominantScript` is checked against the real list — a typo like @@ -286,6 +287,15 @@ analyze(text, { expectedScripts: ['Cyrillic'] }); // real Russian traffic is Latin-dominant, other words already mix scripts, or the word's own script is Latin) **and** when the full UTS #39 skeleton lands in ASCII. Real words like `привет` contain letters with no ASCII prototype and pass through. +- A word already written in Latin needs a UTS #39 **Restricted** character + before its skeleton counts. Nothing there is impersonating Latin — the word + IS Latin — so the skeleton test alone just asks whether the fold happens to + reach ASCII, which it does for ordinary European letters and does so + arbitrarily: `æ` is a ligature and dissolves to `ae`, while `ø` keeps its + stroke as a combining mark and never gets there. `Ægir`, `Þór`, `Straße`, + `cœur`, `ısıtır` and `Hawaiʻi` are all Allowed and pass through; `pɑypal` + (U+0251 IPA ALPHA) is Restricted and is still caught — an intra-Latin + homoglyph no script comparison can see. - `expectedScripts: ['Cyrillic']` marks scripts as normal for the caller's traffic — whole words in them are never flagged; intra-word mixing still is. Declaring it also sharpens detection the other way: a whole word written @@ -322,8 +332,8 @@ platform is the rest of the answer. ## Credits -- Confusables data comes from the Unicode Consortium's - [UTS #39](https://www.unicode.org/reports/tr39/) security tables. +- Confusables and Identifier_Status data both come from the Unicode + Consortium's [UTS #39](https://www.unicode.org/reports/tr39/) security tables. ## Updating the Unicode data @@ -331,6 +341,8 @@ platform is the rest of the answer. npm run generate:data # latest node scripts/generate-confusables.mjs --version 17.0.0 # pinned node scripts/generate-confusables.mjs --file confusables.txt # offline + +node scripts/generate-identifier-status.mjs # Identifier_Status ``` .github/workflows/update-unicode-data.yml is a scheduled GitHub Actions job diff --git a/package.json b/package.json index e476a79..7fee4a7 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "test:coverage": "vitest run --coverage", "publint": "publint", "attw": "attw --pack .", - "generate:data": "node scripts/generate-confusables.mjs", + "generate:data": "node scripts/generate-confusables.mjs && node scripts/generate-identifier-status.mjs", "changeset": "changeset", "version": "changeset version", "release": "pnpm build && changeset publish" diff --git a/scripts/generate-identifier-status.mjs b/scripts/generate-identifier-status.mjs new file mode 100644 index 0000000..77a80db --- /dev/null +++ b/scripts/generate-identifier-status.mjs @@ -0,0 +1,106 @@ +#!/usr/bin/env node +/** + * Regenerates src/data/identifier-status.generated.ts from the Unicode + * Consortium's UTS #39 Identifier_Status data. + * + * Usage: + * node scripts/generate-identifier-status.mjs # fetch /Public/security/latest/ + * node scripts/generate-identifier-status.mjs --version 17.0.0 + * node scripts/generate-identifier-status.mjs --file path/to/IdentifierStatus.txt + * + * The generated file is checked in; this script only needs to run when a new + * Unicode security release ships. A scheduled CI job is the intended caller. + * + * IdentifierStatus.txt lists the ALLOWED characters — those Unicode considers + * appropriate in identifiers. Everything absent from it is Restricted: + * technical, obsolete or limited-use characters that have no business in + * ordinary prose. That distinction is what separates "æ in a Danish surname" + * from "IPA alpha standing in for a latin a". + */ +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const args = process.argv.slice(2); +const getArg = (name) => { + const i = args.indexOf(name); + return i === -1 ? undefined : args[i + 1]; +}; + +async function loadSource() { + const file = getArg('--file'); + if (file) return readFileSync(file, 'utf8'); + const version = getArg('--version') ?? 'latest'; + const url = `https://www.unicode.org/Public/security/${version}/IdentifierStatus.txt`; + const res = await fetch(url); + if (!res.ok) throw new Error(`GET ${url} -> ${res.status}`); + return res.text(); +} + +const source = await loadSource(); + +const versionMatch = source.match(/^#\s*Version:\s*(\S+)/m); +const dateMatch = source.match(/^#\s*Date:\s*([\d-]+)/m); +if (!versionMatch) throw new Error('No "# Version:" header in IdentifierStatus.txt'); + +// Each data line is: ; Allowed or .. ; Allowed +const ranges = []; +for (const line of source.split('\n')) { + const data = line.split('#')[0].trim(); + if (!data) continue; + const [cps, status] = data.split(';').map((s) => s.trim()); + if (status !== 'Allowed' || !cps) continue; + const [from, to] = cps.split('..'); + const start = parseInt(from, 16); + const end = to === undefined ? start : parseInt(to, 16); + if (!Number.isInteger(start) || !Number.isInteger(end) || end < start) { + throw new Error(`Unparseable range: ${cps}`); + } + ranges.push([start, end]); +} +if (ranges.length < 500) { + throw new Error(`Suspiciously few ranges (${ranges.length}) — refusing to write`); +} + +ranges.sort((a, b) => a[0] - b[0]); +// Merge touching/overlapping ranges so the runtime binary search sees a +// strictly increasing, non-overlapping list. +const merged = []; +for (const [start, end] of ranges) { + const last = merged[merged.length - 1]; + if (last && start <= last[1] + 1) last[1] = Math.max(last[1], end); + else merged.push([start, end]); +} + +// Packed as delta-encoded hex pairs: ".", so the +// common case (a short range close to the last one) stays a couple of chars. +let cursor = 0; +const packed = merged + .map(([start, end]) => { + const entry = `${(start - cursor).toString(36)}.${(end - start).toString(36)}`; + cursor = end; + return entry; + }) + .join(';'); +const chunks = packed.match(/.{1,2000}/gs) ?? []; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const outPath = join(__dirname, '..', 'src', 'data', 'identifier-status.generated.ts'); +mkdirSync(dirname(outPath), { recursive: true }); + +writeFileSync( + outPath, + `// AUTO-GENERATED by scripts/generate-identifier-status.mjs — DO NOT EDIT. +// Source: UTS #39 IdentifierStatus.txt (https://www.unicode.org/reports/tr39/) +// Ranges: ${merged.length} (Identifier_Status=Allowed; everything else is Restricted) + +export const IDENTIFIER_STATUS_VERSION = '${versionMatch[1]}'; +export const IDENTIFIER_STATUS_DATE = '${dateMatch?.[1] ?? 'unknown'}'; + +/** Delta-encoded base-36 "." pairs — see the generator. */ +export const ALLOWED_RANGES_PACKED = +${chunks.map((c) => ` '${c}'`).join(' +\n')}; +`, +); + +console.log(`Wrote ${merged.length} allowed ranges (Unicode ${versionMatch[1]}) to ${outPath}`); diff --git a/src/analyze.ts b/src/analyze.ts index 911f066..fd3d546 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -1,4 +1,5 @@ import { foldChar, skeleton } from './confusables'; +import { isRestrictedIdentifierChar } from './identifier-status'; import { analyzeWordScripts, primaryScript, type PseudoScript, type ScriptName } from './scripts'; import { SPOOFING_SIGNALS, @@ -339,6 +340,21 @@ function analyzeToken( // the caller declared the scripts they expect and this word is entirely // OUTSIDE them — a whole word in an unexpected script is spoof evidence // on its own, no Latin context needed. + // + // A word already written in Latin is the exception, and needs one more test. + // Cross-script evidence does not exist there — nothing is impersonating Latin + // because the word IS Latin — so all that remains is whether its skeleton + // happens to reach ASCII. That fires on ordinary European orthography: + // "Ægir" folds to "AEgir" and "ısıtır" to "isitir", both perfectly normal + // words. Worse, it does so arbitrarily — "æ" is a ligature so UTS #39 + // dissolves it to "ae", while "ø" keeps its stroke as a combining mark and + // never reaches ASCII, so "Ægir" was reported and "Ålborg" was not. + // + // Identifier_Status draws the line Unicode intends for exactly this question: + // "æ ø å ß þ œ ı" are Allowed (letters of living alphabets), while "ɑ" (IPA + // alpha) and "fi" (a compatibility ligature) are Restricted. Requiring a + // Restricted character keeps intra-Latin homoglyphs like "pɑypal" — which no + // script check can catch — without indicting every Danish surname. let skeletonConfusable = false; if (!mixed && letters.length >= 2 && !isAsciiWord(token)) { const inExpectedScript = scripts.length > 0 && scripts.every((s) => expectedScripts.has(s)); @@ -350,7 +366,14 @@ function analyzeToken( expectedScripts.size > 0 && scripts.length > 0 && scripts.every((s) => !expectedScripts.has(s)); - if (!inExpectedScript && (latinContext || outsideExpected)) { + // Script-neutral characters (the ʻokina, digits) carry no script evidence + // either way, so a word of Latin + Common letters is still a Latin word. + const writtenInLatin = letters.every((ch) => { + const s = primaryScript(ch); + return s === 'Latin' || s === 'Common' || s === 'Inherited' || s === 'Unknown'; + }); + const carriesRestricted = chars.some((ch) => isRestrictedIdentifierChar(ch)); + if (!inExpectedScript && (latinContext || outsideExpected) && (!writtenInLatin || carriesRestricted)) { const sk = skeleton(token); if (ASCII_PRINTABLE_RE.test(sk) && ASCII_LETTER_RE.test(sk)) { signals.push('confusable_word'); diff --git a/src/data/identifier-status.generated.ts b/src/data/identifier-status.generated.ts new file mode 100644 index 0000000..ebbda72 --- /dev/null +++ b/src/data/identifier-status.generated.ts @@ -0,0 +1,13 @@ +// AUTO-GENERATED by scripts/generate-identifier-status.mjs — DO NOT EDIT. +// Source: UTS #39 IdentifierStatus.txt (https://www.unicode.org/reports/tr39/) +// Ranges: 1612 (Identifier_Status=Allowed; everything else is Restricted) + +export const IDENTIFIER_STATUS_VERSION = '17.0.0'; +export const IDENTIFIER_STATUS_DATE = '2025-08-01'; + +/** Delta-encoded base-36 "." pairs — see the generator. */ +export const ALLOWED_RANGES_PACKED = + '13.0;6.1;2.a;7.p;5.0;2.p;1p.0;9.m;2.u;2.r;3.l;3.3;3.3;2.5;3.7;2.3;3.5;3.9;3.q;3.0;5.0;3.1;4.4;2.0;2.3;4.0;3.1;e.1;2.2;3.0;m.7;9.0;9.3;5.1;9.1;v.3;15.0;8.1;6.1;2.1;2.0;2.0;8.0;5.1;9.0;n.0;2.0;7.0;15.1;1w.4;2.6;f.0;8.0;3.2;9.0;2d.0;2.2;2.0;2.j;2.17;1f.b;2.1t;2.b;2.1;1d.b;3.7;3.f;3.6;f.a;3.d;5.7;3.1;17.1;c.11;b.11;4.0;1y.q;9.1;18.q;3.0;4.h;2.1;b.9;7.0;2.0;2.0;5.m;2.9;5.1;2.0;2.7;2.4;2.0;2.2;3.4;2.j;2.0;p.b;4.2;2a.1;4.0;a.0;2.1;3.2;2.0;4.3;f.1d;66.0;h.0;2.7;i.7;5.0;1m.2;2.6;2.r;2.9;2.7;2.5;2.8;2.0;7.1;f.9;3.5;4.1;2.1;2.2;2.6;4.1;3.l;2.6;2.0;4.3;3.0;2.6;3.1;3.3;o.b;h.0;3.5;5.1;3.l;2.6;2.0;3.0;3.1;3.0;2.4;5.1;3.2;f.0;k.1;h.1;2.8;2.2;2.l;2.6;2.1;2.4;3.0;2.7;2.2;2.2;p.9;i.2;2.6;4.1;3.l;2.6;2.1;3.3;3.0;2.5;4.1;3.2;9.0;9.0;i.0;i.0;2.5;4.2;2.3;4.1;2.0;2.1;4.1;4.2;4.b;5.4;4.2;2.3;1h.1;2.6;3.2;2.m;2.6;2.1;2.4;5.6;2.2;2.3;1h.1;2.6;3.2;2.m;2.6;2.1;2.4;5.5;3.2;2.3;p.9;j.1;2.6;3.2;2.m;2.f;5.5;3.2;2.1;2.0;a.0;z.5;3.1;2.8;4.5;4.3;2.i;2.8;2.0;3.6;4.0;5.5;2.0;2.6;k.0;f.1d;2.6;6.d;3.9;14.1;2.0;3.1;2.0;3.0;7.3;2.6;2.2;2.0;2.0;3.1;2.1;2.2;2.5;2.2;3.4;2.0;2.5;3.9;1e.0;l.9;n.2;2.3;2.3;2.3;2.3;2.3;2.b;9.1;2.0;6.6;4.0;c.2;2.3;2.3;2.3;2.3;2.3;2.0;4.7;2.2;1w.21;h.a;h.l;5.0;1k.0;6.0;3.w;7k.6;2.1s;2.3;3.6;2.0;2.3;3.12;2.0;2.3;3.u;2.0;2.3;3.6;2.0;2.3;3.e;2.v;9.e;2.0;2.3;3.6;2.12;2.h;tj.s;3.3;3.2;3.9;3.n;3.0;2.0;e.9;x3.16;3.2;99.1;5.1;d.1;3.1;h.1;5.3;3.9;f.1;7.1;9.1;3.1;r.1;5.1;b.0;2.2h;4n.f;3.2;1k.0;10.0;9.0;e.0;34u.2;1m.2d;7.1;2.2m;nd.0;18.0;35.0;43.0;13.0;3.0;c.0;i.0;f.0;11.0;b.0;g.0;c.0;la.0;20.0;e.0;2p.1;b.0;4.0;p.0;6.0;q.0;1m.0;a.0;b.0;c.0;5i.0;l.0;2d.0;5n.0;36.0;7b.0;4.0;3t.0;2k.0;au.0;f.0;5.0;29.0;10.0;2g.0;14.0;d4.0;39.0;5.0;y.0;a.0;2u.0;43.0;k.0;7w.0;1y.0;l.0;5e.0;6.0;2b.0;h.0;2v.0;3y.0;5h.0;1f.0;3.0;5.1;2.1;l.0;4.0;n.1;2j.0;4m.0;cf.0;6.0;4.0;4.0;o.6;34.6;45.0;2a.h;2.l;2.1p;2.f;2.r;2.b;2.n;2.e;2.p;2.f;4.0;2.0;2.7;2.d;2.g;2.7;2.2;2.0;2.7;2.2;2.17;2.p;2.0;2.i;2.p;2.7;2.0;2.2;2.k;2.o;3.h;2.y;2.2;2.3;2.3;2.e;2.4;2.y;2.3;2.g;2.2;2.6;4.m;2.8;2.j;2.5;2.0;3.7;2.7;2.1;2.y;2.b;2.1;2.l;2.6;2.3;2.2;2.' + + '3;2.6;2.2;2.c;2.a;2.3;3.8;2.h;2.0;2.8;2.9;2.0;2.6;2.e;3.a;2.b;2.0;2.2;2.2n;2.h;2.h;2.a;2.7;2.1g;2.2;2.2;2.0;2.1;2.1;2.b;2.2;2.7;2.j;3.i;2.1;2.1;2.7;2.0;2.7;2.1;3.a;2.n;2.0;2.b;2.1;2.h;2.v;2.7;3.2;2.9;2.9;2.8;2.c;3.7;2.m;3.o;2.6;2.14;2.f;2.a;2.v;2.15;2.1r;2.t;2.1;3.18;2.n;2.o;2.2s;2.m;2.g;3.j;2.9;2.a;2.3;2.6;2.5;2.4;2.3;2.7;2.0;2.i;2.6;2.9;2.1;2.9;2.6;2.q;2.j;2.8;2.b;2.e;2.0;2.1;3.7;2.a;2.4;2.8;2.1;2.n;2.4;2.3;2.b;3.0;2.9;2.j;2.7;2.4;2.d;2.g;2.w;2.5;2.8;2.1m;3.1i;2.4;2.8;2.4;3.e;2.s;2.j;2.a;2.8;2.1;3.7;2.1;2.f;2.3;2.d;2.a;2.2;2.1;2.7;2.0;2.e;2.e;2.1;2.x;2.0;2.2i;2.1a;2.1n;2.0;2.4;2.2;2.y;2.3;2.k;2.0;2.k;2.4;2.s;2.12;2.q;3.o;2.1;2.16;2.b;2.7;2.1;2.c;3.c;2.8;2.3;2.6;2.8;2.n;2.7;2.3;2.3;2.a;2.g;2.10;2.p;2.6;2.1;2.1;2.b;2.y;2.15;2.19;2.3;2.q;2.6;2.m;2.1g;2.2;2.a;2.m;2.2;2.7;2.x;2.h;2.b;3.5;2.6;2.4;2.9;2.h;2.1;2.0;2.o;2.q;2.1g;2.7;2.1r;2.12;2.4;2.15;2.c;2.j;2.17;2.0;2.28;2.9;2.7;3.a;2.g;2.a;2.b;2.5;2.8;2.e;2.11;2.1d;2.9;2.e;2.1;2.r;2.0;2.g;2.17;2.2;2.6;2.5;2.m;2.3;2.3;2.8;2.b;2.h;2.0;2.0;2.1;3.11;2.p;2.2;2.5;2.f;2.3;2.r;2.0;2.m;2.0;2.l;2.1;2.k;2.1a;2.2;2.s;2.1g;2.e;2.1r;2.a;2.u;2.b;2.o;2.4;2.y;2.1a;2.6;2.1o;2.2w;3.b;2.f;2.y;2.6;2.p;2.c;2.5;2.g;2.b;2.n;2.5;2.2;2.7;2.6;2.j;2.7;2.m;2.0;2.m;2.3;2.s;2.b;2.e;2.p;2.7;2.8;3.6;2.5;2.2;2.6;2.4;2.d;2.3;2.n;2.j;2.9;3.e;2.3;2.h;2.1;2.19;2.8;2.0;2.c;3.x;2.1;2.2;2.6;2.e;2.h;2.3;2.g;2.9;2.3;2.1o;2.v;2.c;2.11;2.0;2.k;2.0;2.w;2.6;2.i;2.g;2.7;2.12;2.0;2.2;2.q;2.p;2.2;2.a;2.h;2.9;2.k;2.7;2.1;2.y;2.1f;2.d;2.28;2.1;2.s;2.f;3.1;4.16;2.n;2.6;2.h;2.h;2.g;2.g;2.0;2.k;2.2;2.3;2.8;2.2;2.i;2.b;2.0;2.2;2.3;2.d;2.f;2.z;2.b;2.h;2.5;2.9;2.l;3.3;2.9;2.2;2.0;2.1;2.2;2.6;2.1l;2.1b;2.6;2.1;2.4;2.e;2.g;2.f;2.d;2.7;2.11;2.0;2.1c;2.1i;2.q;2.c;2.o;2.6;2.23;2.e;2.j;2.o;3.3;2.1y;3.0;3.2;2.a;2.4;2.g;2.w;2.u;2.d;4.c;2.8;2.1w;2.6;2.v;2.5;2.1h;2.4;2.0;2.19;2.t;2.0;2.3;2.m;2.c;2.9;2.1;2.w;2.6;2.h;2.1;2.1e;2.2;2.1;2.y;3.6;2.4;2.a;2.8;2.2;2.d;2.6;2.2;2.t;2.13;2.8;2.q;2.d;3.x;2.i;2.0;2.u;2.7;2.e;2.3;2.9;2.k;2.5;2.u;2.5;2.f;2.4;2.2;2.0;2.7;2.2;3.8;2.3;2.f;2.7;2.0;2.3;2.2c;' + + '2.h;2.1u;2.c;2.m;2.15;2.1z;2.z;2.1c;2.6;2.r;2.v;2.3;2.7;2.0;2.d;2.x;2.7;2.g;2.5;2.14;2.0;2.1;2.t;2.1f;2.v;2.f;2.1;2.7;2.1;2.e;2.8;2.8;2.y;2.c;2.z;2.g;2.1o;2.m;3.a;2.2;2.s;2.o;2.i;3.1;2.1;2.9;2.1;2.9;2.8;2.c;2.g;2.j;2.0;2.a;2.h;2.8;2.i;2.1;2.w;2.1c;2.c;2.o;2.1;2.f;2.9;2.5;2.0;2.c;3.19;2.g;2.a;2.2;2.3;2.15;2.g;2.1k;2.3;2.9;2.s;2.6;2.6;2.6;2.3;2.6;2.8;3.g;2.2;2.x;2.h;2.0;2.5;3.e;3.l;2.9;2.h;2.6;2.1;2.3;2.6;2.1;2.14;2.4;2.0;2.x;2.a;2.0;2.14;2.4;2.3;2.h;2.m;3.s;2.0;4.3;2.2;2.x;2.a;2.5;2.9;2.m;2.6;2.0;2.n;2.f;2.26;2.0;2.9;2.0;2.1y;2.v;2.h;3.x;2.1;2.6;2.n;2.f;2.a;2.5;2.g;2.11;2.5;3.4;2.0;2.0;3.k;2.1;2.y;2.5;2.1;2.p;2.3;2.b;2.8;2.i;2.8;2.h;2.1;2.a;2.2;2.z;2.4;2.g;2.4;2.g;4.n;2.1t;2.7;2.1j;2.4c;3.3;2.3;2.h;2.15;2.a;2.5;2.0;2.p;2.w;2.5;2.5;2.10;2.z;2.b;2.2;2.10;2.w;2.4;2.4;2.g;4.9;2.4;2.10;2.y;3.1a;3.25;2.a;2.4;2.1f;2.n;2.9;3.x;2.f;2.r;2.y;2.11;2.10;2.1a;2.o;2.c;2.4;2.u;2.z;2.2v;2.12;2.5;2.11;2.0;3.6;2.1j;2.15;2.g;2.w;2.a;2.14;2.m;2.9;2.w;3.n;2.2;2.19;2.i;2.t;2.7;2.f;2.w;2.3;2.7;2.5;2.8;3.0;2.3;2.6;2.o;2.u;2.s;2.i;2.7;2.c;2.4;2.19;2.9;2.5;2.k;2.u;2.g;2.c;2.6;2.16;2.r;2.9;2.1r;2.k;2.l;2.9;2.p;2.b;2.b;2.f;2.2;2.1;2.p;2.2y;2.1;2.2;2.k;2.a;2.j;2.1r;2.e;2.6;2.e;2.1;2.2;2.3;2.l;2.2;2.5;3.1c;2.5;2.2;2.2;2.6;3.c;2.3;2.5;3.8;2.g;2.c;2.n;2.i;2.3;2.9;2.l;2.e;2.6;2.n;2.q;2.1n;2.7;2.4;2.5;2.w;3.5;2.a;2.5;2.3;2.4;2.4;2.0;2.a;2.4b;2.6;2.2;2.b;4.3;2.q;2.9;2.2;2.2;2.2;2.e;2.9;2.z;2.9;2.0;2.f;2.c;2.8;2.e;3.2w;2.14;2.o;2.z;2.q;2.8;2.11;2.i;2.7;2.t;2.5;2.0;2.0;2.w;2.5;2.f;2.1;3.8;2.1;2.j;2.l;2.t;2.s;2.b;2.5;2.2i;2.5;2.d;2.e;2.5;2.r;2.1t;2.0;2.9;2.7;2.4;2.2r;2.v;3.3;2.y;2.i;2.1c;3.q;2.w;2.o;2.1;2.f;2.k;2.3;2.q;2.d;2.y;2.a;2.1;2.v;2.10;3.l;2.1;2.o;2.t;2.l;2.5;2.5;2.1k;2.i;3.1b;2.4;2.1;2.2;2.2;2.3;2.5;2.0;2.8;2.f;2.2;2.9;2.e;2.z;2.h;2.d;2.j;2.3;2.0;3.3;2.t;2.9;2.s;2.d;2.6z;3.3;2.5;2.g;2.q;3.2;2.7;2.m;2.2;2.1y;3.1;2.j;2.m;3.w;2.0;2.2;2.0;3.8;2.8;2.2;2.1;2.0;2.a;2.2;2.4;2.19;2.0;3.j;2.m;2.1;2.1;3.m;3.1;2.7;3.a;2.8;2.a;2.g;2.g;2.0;2.g;2.c;2.5;3.2;2.7;2.0;2.g;2.14;2.x;2.4;2.3;2.7;2.5;2.2;2.2;2.3;2.d;2.' + + 'e;2.0;2.25;2.h;2.o;2.6;2.3;2.5;2.1;2.s;2.w;2.7;2.k;2.1;2.2o;2.v;2.4;2.3;2.8;2.p;2.5;3.x;2.k;2.5;2.3;2.4;2.1w;2.i;2.9;2.d;2.7;2.2;2.2;3.5;2.e;2.0;2.5;2.9;2.5;2.n;2.7;2.1;2.e;2.5;2.9;2.e;2.l;2.1;2.b;2.13;2.1e;2.8;3.0;2.1p;2.b;2.0;2.e;2.1;2.l;2.3z;3.2;2.k;2.c;2.5;2.1b;2.12;2.n;2.d;2.1;2.14;2.g;2.g;2.6;2.4;2.2l;2.i;3.6;2.4;2.2;2.7;3.3;2.1l;2.3;2.g;2.c;2.1;2.g;2.t;2.c;2.o;2.1e;2.0;2.1;1k8.0;t.0;k1.0;at.8mb;bq6.0;2.0;1l.0;1604.6;2.3;2.1;2.e;64w.0;z.0;20.0;yi.0;11.0;u.0;1l.0;6.0;1s.0;2v.0;3.0;3z.1;2w.0;12.0;5.0;1h.0;y.1;1f.1;u.0;2w.0;8.0;1a.0;36.0;j.0;6.1;5.0;1y.0;8.0;7k.0;2y6.0;1da.0;m8.0;32.0;3r.0;3r.0;4.0;31.0;1y.0;1w.0;r.0;98.0;2ro.0;1lo.0;1r9.0;1e.0;n5.0;3e5.0;12s.0;fa.0;8.0;37o.0;1en.0;4q.0;63.0;1yg.0;3.0;5.0;3ba.0'; diff --git a/src/identifier-status.ts b/src/identifier-status.ts new file mode 100644 index 0000000..4c4a8a5 --- /dev/null +++ b/src/identifier-status.ts @@ -0,0 +1,49 @@ +import { ALLOWED_RANGES_PACKED } from './data/identifier-status.generated'; + +/** + * UTS #39 Identifier_Status. Unicode publishes the set of characters it + * considers appropriate in identifiers ("Allowed"); everything else is + * "Restricted" — technical notation, obsolete letters, limited-use scripts. + * + * This is the distinction that tells a legitimate non-ASCII word from a + * disguised one *within a single script*, where script analysis has nothing to + * say. "æ" in a Danish surname and "ɑ" (IPA alpha) standing in for "a" are + * both lone Latin letters whose skeleton is ASCII; only their status differs. + */ +const starts: number[] = []; +const ends: number[] = []; + +{ + let cursor = 0; + for (const entry of ALLOWED_RANGES_PACKED.split(';')) { + const dot = entry.indexOf('.'); + const start = cursor + parseInt(entry.slice(0, dot), 36); + const end = start + parseInt(entry.slice(dot + 1), 36); + starts.push(start); + ends.push(end); + cursor = end; + } +} + +/** True when Unicode marks this code point as Allowed in identifiers. */ +export function isAllowedIdentifierChar(cp: number): boolean { + // Ranges are sorted and non-overlapping, so the candidate is the last range + // whose start is <= cp. + let lo = 0; + let hi = starts.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (starts[mid]! > cp) hi = mid - 1; + else if (ends[mid]! < cp) lo = mid + 1; + else return true; + } + return false; +} + +/** + * True when a character is Restricted — the inverse of the above, named for + * how it reads at the call site. + */ +export function isRestrictedIdentifierChar(ch: string): boolean { + return !isAllowedIdentifierChar(ch.codePointAt(0)!); +} diff --git a/test/analyze.test.ts b/test/analyze.test.ts index c609ac9..4ad838f 100644 --- a/test/analyze.test.ts +++ b/test/analyze.test.ts @@ -114,6 +114,7 @@ describe('analyze — legitimate content', () => { ['typographic apostrophes', 'Sounds good, I’ll call — don’t worry, it’s Kaysie’s job.'], ['& apos; in HTML', 'Sounds good, I'll call — don't worry, it's Kaysie's job.'], ['test', 'Hansen&sons'], + ['danish name', "Hi Ægir, I hope you're well."], ['empty string', ''], ]; @@ -132,6 +133,42 @@ describe('analyze — legitimate content', () => { expect(r.signals.mixed_script).toBe(true); }); + // A word already written in Latin cannot be impersonating Latin, so the only + // thing the skeleton test measures there is whether the fold happens to reach + // ASCII — which it does for ordinary European letters, and arbitrarily: "æ" + // is a ligature and dissolves to "ae", while "ø" keeps its stroke as a + // combining mark and never gets there. Identifier_Status replaces that coin + // flip with the distinction Unicode defines for the purpose. + describe('Latin words are judged on Identifier_Status, not skeleton luck', () => { + it.each([ + ['Nordic ligature and stroke alike', 'Ægir sailed to Ålborg'], + ['thorn and eth', 'Þór og Ðagur'], + ['German sharp s', 'Straße und Größe'], + ['French ligature', 'Le cœur et la sœur'], + ['Turkish dotless i', 'Işık ılık ısıtır'], + ['Hawaiian ʻokina', 'Hawaiʻi and Oʻahu'], + ['a name mangled by a codepage swap', 'offer for Ayalaæs Lashes'], + ])('leaves %s alone', (_name, text) => { + expect(analyze(text).spoofed).toBe(false); + }); + + it('still catches an intra-Latin homoglyph, which no script check can see', () => { + // U+0251 LATIN SMALL LETTER ALPHA is Latin script — mixed_script and the + // cross-script skeleton path are both blind to it. Identifier_Status + // marks it Restricted, which is the only thing separating it from "æ". + const r = analyze('Verify your pɑypal account'); + expect(r.signals.confusable_word).toBe(true); + expect(r.normalized).toBe('Verify your paypal account'); + }); + + it('does not weaken the cross-script case, where letters are Allowed too', () => { + // Cyrillic "а"/"о" are Allowed — they are ordinary Russian. The gate must + // apply ONLY to Latin words, or "раураl" would walk straight through. + expect(analyze('Verify your раураl account').spoofed).toBe(true); + expect(analyze('НОТ deal today').spoofed).toBe(true); + }); + }); + it('does not call a styled word confusable unless the whole word folds to ASCII', () => { // "𝗉𝗋é" has two styled letters that individually fold to ASCII, but the // token folds to "pré" — still non-ASCII, so it is not a disguised word. From 479f3329f87a2cf658346ccb6a4ae449d7a67481 Mon Sep 17 00:00:00 2001 From: CUDE Date: Mon, 20 Jul 2026 17:25:44 +0200 Subject: [PATCH 5/5] style: apply prettier formatting Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 4 ++-- src/analyze.ts | 6 +++++- src/index.ts | 7 +------ src/types.ts | 7 +------ 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index f9ea427..7ccff53 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ analyze('Ship it 🎉 👨\u200D👩\u200D👧 ℹ\uFE0F').spoofed; // false — | `invisible` | Characters that render as nothing, in or between words | zero-width, bidi, tag chars, fillers | | `zalgo` | Combining marks stacked beyond orthographic depth (≥3 per base) | `Z̸̢̬a̛lg̕o` | | `illegal` | Control or non-character code points anywhere in text | `NUL`, `U+0085`, `U+FFFE` | -| `encoding_damage` | Decode damage — reported, but never sets `spoofed` (see above) | `Jos��` (a mangled `José`) | +| `encoding_damage` | Decode damage — reported, but never sets `spoofed` (see above) | `Jos��` (a mangled `José`) | What `invisible` covers: format characters (zero-width space/joiner, word joiner, …), bidi controls incl. the Trojan Source overrides, tag characters, @@ -227,7 +227,7 @@ against a name instead of a hardcoded string. | Export | What it is | | -------------------------------- | ----------------------------------------------------------------------------------------------------- | -| `SPOOF_SIGNALS` | All six signal names — the keys of `result.signals` | +| `SPOOF_SIGNALS` | All six signal names — the keys of `result.signals` | | `SCRIPT_NAMES` | Every script the classifier can name — the possible values of `dominantScript` and `words[].scripts` | | `SUPPORTED_SCRIPTS` | The subset of `SCRIPT_NAMES` this runtime's Unicode tables support (all of them, on a current engine) | | `PSEUDO_SCRIPTS` | `Common`, `Inherited`, `Unknown` — classifications that are not scripts and never appear in a finding | diff --git a/src/analyze.ts b/src/analyze.ts index fd3d546..cccee8e 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -373,7 +373,11 @@ function analyzeToken( return s === 'Latin' || s === 'Common' || s === 'Inherited' || s === 'Unknown'; }); const carriesRestricted = chars.some((ch) => isRestrictedIdentifierChar(ch)); - if (!inExpectedScript && (latinContext || outsideExpected) && (!writtenInLatin || carriesRestricted)) { + if ( + !inExpectedScript && + (latinContext || outsideExpected) && + (!writtenInLatin || carriesRestricted) + ) { const sk = skeleton(token); if (ASCII_PRINTABLE_RE.test(sk) && ASCII_LETTER_RE.test(sk)) { signals.push('confusable_word'); diff --git a/src/index.ts b/src/index.ts index 9999aaa..fa0ded3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,4 @@ -export { - analyze, - FORMAT_CHAR_SCRIPTS, - ZALGO_MARK_RUN, - ZERO_WIDTH_INERT_RUN, -} from './analyze'; +export { analyze, FORMAT_CHAR_SCRIPTS, ZALGO_MARK_RUN, ZERO_WIDTH_INERT_RUN } from './analyze'; export { skeleton } from './confusables'; export { UNICODE_VERSION, DATA_DATE } from './data/confusables.generated'; export { diff --git a/src/types.ts b/src/types.ts index b8ef4b0..c7fefc1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,12 +1,7 @@ import type { ScriptName } from './scripts'; export type SpoofSignal = - | 'mixed_script' - | 'confusable_word' - | 'invisible' - | 'zalgo' - | 'illegal' - | 'encoding_damage'; + 'mixed_script' | 'confusable_word' | 'invisible' | 'zalgo' | 'illegal' | 'encoding_damage'; export const SPOOF_SIGNALS: readonly SpoofSignal[] = [ 'mixed_script',