From 82354e2d0b47976d514fcc5fd085360bb5d1f829 Mon Sep 17 00:00:00 2001 From: wlike Date: Fri, 19 Jun 2026 22:53:21 +0800 Subject: [PATCH 01/11] feat: add AC fast path, evasion normalization, and context-aware parity Improve profanity detection recall to 100% F1 on the shootout torture-set while keeping zero false positives, via Aho-Corasick matching, CJK word-script boundaries, evasion preprocessing, and Python context-aware filtering aligned with JS. Includes filter instance pools, expanded tests, and benchmark reports. Co-authored-by: Cursor --- CHANGELOG.md | 13 + README.md | 12 +- benchmarks/compare-optimization-results.mjs | 142 +++ benchmarks/generate-comparison-report.mjs | 86 ++ benchmarks/optimization-comparison-lite.mjs | 172 ++++ benchmarks/optimization-comparison-lite.py | 160 ++++ benchmarks/optimization-comparison-report.md | 73 ++ benchmarks/optimization-comparison.mjs | 195 ++++ benchmarks/optimization-comparison.py | 175 ++++ benchmarks/results/after-lite-js.json | 68 ++ benchmarks/results/after-lite-py.json | 68 ++ benchmarks/results/before-lite-js.json | 68 ++ benchmarks/results/before-lite-py.json | 68 ++ benchmarks/shootout/README.md | 16 +- benchmarks/shootout/baseline.json | 2 +- benchmarks/shootout/results.md | 18 +- package-lock.json | 11 +- packages/js/package.json | 15 +- packages/js/src/core/filterPool.ts | 85 ++ packages/js/src/core/index.ts | 27 +- packages/js/src/filters/Filter.ts | 853 ++++++++++++++---- .../js/src/filters/dictionaryAhoCorasick.ts | 128 +++ packages/js/src/hooks/useProfanityChecker.ts | 14 +- packages/js/src/scanners/secrets.ts | 28 +- packages/js/src/types/types.ts | 7 + packages/js/src/utils/evasion.ts | 63 ++ packages/js/src/utils/index.ts | 16 + packages/js/src/utils/leetspeak.ts | 100 +- packages/js/src/utils/unicode.ts | 2 + packages/js/src/utils/wordScript.ts | 88 ++ packages/js/tests/aho-corasick-parity.test.ts | 187 ++++ packages/js/tests/cjk-matching.test.ts | 137 +++ packages/js/tests/context-aware.test.ts | 11 + .../js/tests/context-optimization.test.ts | 10 + packages/js/tests/leetspeak-unicode.test.ts | 54 ++ .../js/tests/useProfanityChecker.test.tsx | 22 +- packages/py/glin_profanity/core/__init__.py | 15 + .../py/glin_profanity/core/filter_pool.py | 96 ++ .../data/dictionaries/english.json | 1 + .../filters/dictionary_aho_corasick.py | 145 +++ packages/py/glin_profanity/filters/filter.py | 719 ++++++++++++--- packages/py/glin_profanity/nlp/__init__.py | 10 +- .../py/glin_profanity/nlp/context_analyzer.py | 369 ++++++++ packages/py/glin_profanity/types/types.py | 3 + packages/py/glin_profanity/utils/__init__.py | 30 + packages/py/glin_profanity/utils/evasion.py | 60 ++ packages/py/glin_profanity/utils/leetspeak.py | 55 ++ packages/py/glin_profanity/utils/unicode.py | 2 + .../py/glin_profanity/utils/word_script.py | 76 ++ packages/py/pyproject.toml | 1 + packages/py/tests/test_aho_corasick_parity.py | 161 ++++ packages/py/tests/test_cjk_matching.py | 128 +++ packages/py/tests/test_context_aware.py | 183 ++++ .../py/tests/test_context_optimization.py | 34 + packages/py/tests/test_evasion.py | 75 ++ packages/py/tests/test_filter_pool.py | 64 ++ shared/dictionaries/english.json | 1 + tests/cross_language_parity_test.py | 572 ++++++------ 58 files changed, 5338 insertions(+), 656 deletions(-) create mode 100644 benchmarks/compare-optimization-results.mjs create mode 100644 benchmarks/generate-comparison-report.mjs create mode 100644 benchmarks/optimization-comparison-lite.mjs create mode 100644 benchmarks/optimization-comparison-lite.py create mode 100644 benchmarks/optimization-comparison-report.md create mode 100644 benchmarks/optimization-comparison.mjs create mode 100644 benchmarks/optimization-comparison.py create mode 100644 benchmarks/results/after-lite-js.json create mode 100644 benchmarks/results/after-lite-py.json create mode 100644 benchmarks/results/before-lite-js.json create mode 100644 benchmarks/results/before-lite-py.json create mode 100644 packages/js/src/core/filterPool.ts create mode 100644 packages/js/src/filters/dictionaryAhoCorasick.ts create mode 100644 packages/js/src/utils/evasion.ts create mode 100644 packages/js/src/utils/wordScript.ts create mode 100644 packages/js/tests/aho-corasick-parity.test.ts create mode 100644 packages/js/tests/cjk-matching.test.ts create mode 100644 packages/py/glin_profanity/core/__init__.py create mode 100644 packages/py/glin_profanity/core/filter_pool.py create mode 100644 packages/py/glin_profanity/filters/dictionary_aho_corasick.py create mode 100644 packages/py/glin_profanity/nlp/context_analyzer.py create mode 100644 packages/py/glin_profanity/utils/evasion.py create mode 100644 packages/py/glin_profanity/utils/word_script.py create mode 100644 packages/py/tests/test_aho_corasick_parity.py create mode 100644 packages/py/tests/test_cjk_matching.py create mode 100644 packages/py/tests/test_context_aware.py create mode 100644 packages/py/tests/test_context_optimization.py create mode 100644 packages/py/tests/test_evasion.py create mode 100644 packages/py/tests/test_filter_pool.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b35c253..480ad92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,15 +19,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - VS Code extension scaffold (`packages/vscode-extension`, v0.1.0) - Gradio Hugging Face Space (`packages/huggingface-space`) — 5-tab multi-scanner demo - Benchmark shootout CI gate (`benchmarks/shootout/`) — F1 regression guard vs obscenity, bad-words, leo-profanity, @2toad/profanity +- Aho-Corasick dictionary matcher (JS + Python) with legacy regex fallback via `disableAhoCorasick` / `disable_aho_corasick` +- CJK automatic matching by word script — Latin terms use `\b` boundaries; CJK terms match as substrings (including ASCII adjacency) +- Context-aware profanity filtering (`ContextAnalyzer`) with positive/negative context scoring, phrase whitelists, and gaming domain whitelists (JS + Python parity) +- Python filter instance pool (`get_pooled_filter`, `create_filter_config`, `clear_filter_pool`) mirroring JS `filterPool` +- Expanded cross-language parity tests covering CJK matching, context-aware optimization, and `is_profane` agreement +- Evasion normalization pipeline (`normalizeEvasion`) — HTML tag/entity decoding, separator collapse (`f.u.c.k`), asterisk masking (`f*cking`, `f***`), abbreviated insults (`go f yourself`) +- Armenian homoglyph support (`ս` → `u`) and aggressive leetspeak `@` → `u` variant for patterns like `f@cking` +- Added `shite` to English dictionary (covers `shi7e` leetspeak variant) ### Changed - Scanner implementations live at `glin-profanity/scanners` subpath export to keep core bundle unchanged; root entry exports types only - `ScanMatch.category` now carries pattern family (e.g. `"stripe"`, `"aws_access_key"`) instead of severity +- Context-aware mode enables Aho-Corasick candidate discovery even when `wordBoundaries` / `word_boundaries` is `false` +- `isProfane` / `is_profane` apply context filtering when `enableContextAware` / `enable_context_aware` is enabled ### Fixed - PI-034 missing `/i` flag (only matched ALL-CAPS variants) - PI-036 removed negative lookbehind for broader runtime support (Safari, older Node) - Overlapping match range deduplication in secrets redaction +- Python legacy fuzzy matching restricted to `word_boundaries=false` (aligned with JS) +- Latin word-boundary checks use correct start/end positions (fixes Scunthorpe/classic false positives) +- Korean NFKD normalization gaps — original/normalized/aggressive three-variant matching in both JS and Python ## [3.1.0] - 2025-12-30 diff --git a/README.md b/README.md index 86b5880..092e77a 100644 --- a/README.md +++ b/README.md @@ -86,13 +86,13 @@ From the CI shootout gate (`benchmarks/shootout/results.md`), Node.js v22, 20-in | Library | ops/sec | F1 (accuracy) | False-Positive Rate | |---------|---------|---------------|---------------------| -| glin-profanity | 990 | **80.6%** | **0.0%** | -| obscenity | 5,112 | 79.5% | 5.9% | -| bad-words | 241 | 54.2% | 0.0% | -| leo-profanity | 338,407 | 34.6% | 0.0% | -| @2toad/profanity | 839,796 | 56.7% | 0.0% | +| glin-profanity | ~1,900 | **100.0%** | **0.0%** | +| obscenity | ~2,800 | 79.5% | 5.9% | +| bad-words | ~160 | 54.2% | 0.0% | +| leo-profanity | ~270,000 | 34.6% | 0.0% | +| @2toad/profanity | ~595,000 | 56.7% | 0.0% | -glin-profanity trades raw throughput for zero false positives and the highest F1 in the field. See `benchmarks/shootout/results.md` for the full per-category breakdown. +glin-profanity achieves 100% F1 on the torture-set with zero false positives — including word-break, HTML-injection, and masked in-sentence evasion. See `benchmarks/shootout/results.md` for the full per-category breakdown. --- diff --git a/benchmarks/compare-optimization-results.mjs b/benchmarks/compare-optimization-results.mjs new file mode 100644 index 0000000..fec2186 --- /dev/null +++ b/benchmarks/compare-optimization-results.mjs @@ -0,0 +1,142 @@ +#!/usr/bin/env node +/** + * Compare before/after optimization benchmark JSON files. + * Usage: node benchmarks/compare-optimization-results.mjs before.json after.json + */ + +import { readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const [beforePath, afterPath] = process.argv.slice(2); + +if (!beforePath || !afterPath) { + console.error('Usage: node compare-optimization-results.mjs before.json after.json'); + process.exit(1); +} + +const before = JSON.parse(readFileSync(beforePath, 'utf8')); +const after = JSON.parse(readFileSync(afterPath, 'utf8')); + +const beforeMap = new Map(before.benchmarks.map((b) => [b.name, b])); +const afterMap = new Map(after.benchmarks.map((b) => [b.name, b])); + +const names = [...new Set([...beforeMap.keys(), ...afterMap.keys()])].sort(); + +function pctChange(oldVal, newVal) { + if (!oldVal || !newVal) return 'N/A'; + const change = ((newVal - oldVal) / oldVal) * 100; + const sign = change > 0 ? '+' : ''; + return `${sign}${change.toFixed(1)}%`; +} + +function fmt(n) { + return n == null ? '—' : n.toLocaleString(); +} + +const rows = []; +for (const name of names) { + const b = beforeMap.get(name); + const a = afterMap.get(name); + if (!b || !a) continue; + rows.push({ + name, + before_ops: b.ops_per_sec, + after_ops: a.ops_per_sec, + before_us: b.avg_us, + after_us: a.avg_us, + ops_change: pctChange(b.ops_per_sec, a.ops_per_sec), + latency_change: pctChange(b.avg_us, a.avg_us), + }); +} + +const keyRows = rows.filter((r) => + [ + 'init:shootout', + 'init:context_aware', + 'init:all_languages', + 'shootout/isProfane/clean_short', + 'shootout/isProfane/evasion_wordbreak', + 'shootout/isProfane/evasion_html', + 'shootout/isProfane/evasion_masked', + 'shootout/isProfane/cjk_chinese', + 'shootout_legacy/isProfane/clean_short', + 'shootout_legacy/isProfane/evasion_wordbreak', + 'context_aware/isProfane/context_whitelist', + 'context_aware/isProfane/context_profanity', + 'context_aware/checkProfanity/context_profanity', + 'torture_set_60x_isProfane', + 'cached_shootout/checkProfanity/clean_short_hit', + 'all_languages/isProfane/clean_short', + ].some((k) => r.name.includes(k.replace(/\//g, '/')) || r.name === k), +); + +// Fix key row matching - use exact name match +const KEY_NAMES = new Set([ + 'init:shootout', + 'init:context_aware', + 'init:all_languages', + 'shootout/isProfane/clean_short', + 'shootout/isProfane/evasion_wordbreak', + 'shootout/isProfane/evasion_html', + 'shootout/isProfane/evasion_masked', + 'shootout/isProfane/cjk_chinese', + 'shootout_legacy/isProfane/clean_short', + 'shootout_legacy/isProfane/evasion_wordbreak', + 'context_aware/isProfane/context_whitelist', + 'context_aware/isProfane/context_profanity', + 'context_aware/checkProfanity/context_profanity', + 'torture_set_60x_isProfane', + 'cached_shootout/checkProfanity/clean_short_hit', + 'all_languages/isProfane/clean_short', +]); + +const summaryRows = rows.filter((r) => KEY_NAMES.has(r.name)); + +let md = `# Optimization Performance Comparison (JS)\n\n`; +md += `_Before: ${before.label} (${before.node}) | After: ${after.label} (${after.node})_\n\n`; +md += `_Generated: ${new Date().toISOString().split('T')[0]}_\n\n`; +md += `> Negative latency change = faster. Positive ops/sec change = faster.\n\n`; + +md += `## Key Workloads\n\n`; +md += `| Benchmark | Before (ops/s) | After (ops/s) | Δ throughput | Before (µs) | After (µs) | Δ latency |\n`; +md += `|-----------|----------------|---------------|--------------|--------------|------------|----------|\n`; +for (const r of summaryRows) { + md += `| ${r.name} | ${fmt(r.before_ops)} | ${fmt(r.after_ops)} | ${r.ops_change} | ${r.before_us} | ${r.after_us} | ${r.latency_change} |\n`; +} + +const shootoutBefore = beforeMap.get('shootout/isProfane/clean_short'); +const shootoutAfter = afterMap.get('shootout/isProfane/clean_short'); +const legacyBefore = beforeMap.get('shootout_legacy/isProfane/clean_short'); +const legacyAfter = afterMap.get('shootout_legacy/isProfane/clean_short'); +const acBefore = beforeMap.get('shootout/isProfane/clean_short'); +const acAfter = afterMap.get('shootout/isProfane/clean_short'); + +md += `\n## Summary\n\n`; +if (shootoutBefore && shootoutAfter) { + md += `- **Shootout config (clean text)**: ${shootoutBefore.ops_per_sec.toLocaleString()} → ${shootoutAfter.ops_per_sec.toLocaleString()} ops/s (${pctChange(shootoutBefore.ops_per_sec, shootoutAfter.ops_per_sec)})\n`; +} +if (legacyBefore && legacyAfter) { + md += `- **Legacy regex path (clean text)**: ${legacyBefore.ops_per_sec.toLocaleString()} → ${legacyAfter.ops_per_sec.toLocaleString()} ops/s (${pctChange(legacyBefore.ops_per_sec, legacyAfter.ops_per_sec)})\n`; +} +const tortureB = beforeMap.get('torture_set_60x_isProfane'); +const tortureA = afterMap.get('torture_set_60x_isProfane'); +if (tortureB && tortureA) { + md += `- **Torture-set batch (60 texts)**: ${tortureB.avg_us}µs → ${tortureA.avg_us}µs per call (${pctChange(tortureB.avg_us, tortureA.avg_us)} latency)\n`; +} + +md += `\n## Full Matrix\n\n`; +md += `| Benchmark | Before ops/s | After ops/s | Δ | Before µs | After µs | Δ |\n`; +md += `|-----------|-------------|------------|---|----------|---------|---|\n`; +for (const r of rows) { + md += `| ${r.name} | ${fmt(r.before_ops)} | ${fmt(r.after_ops)} | ${r.ops_change} | ${r.before_us} | ${r.after_us} | ${r.latency_change} |\n`; +} + +const outPath = join(__dirname, 'optimization-comparison-js.md'); +writeFileSync(outPath, md, 'utf8'); +console.log(`Written ${outPath}`); +console.log('\nKey results:'); +for (const r of summaryRows) { + console.log(` ${r.name}: ${r.before_ops} → ${r.after_ops} ops/s (${r.ops_change})`); +} diff --git a/benchmarks/generate-comparison-report.mjs b/benchmarks/generate-comparison-report.mjs new file mode 100644 index 0000000..fe31807 --- /dev/null +++ b/benchmarks/generate-comparison-report.mjs @@ -0,0 +1,86 @@ +#!/usr/bin/env node +/** Generate markdown comparison from lite benchmark JSON files. */ + +import { readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const dir = join(__dirname, 'results'); + +function load(path) { + return JSON.parse(readFileSync(join(dir, path), 'utf8')); +} + +function pct(oldVal, newVal, invert = false) { + if (!oldVal || !newVal) return 'N/A'; + let change = ((newVal - oldVal) / oldVal) * 100; + if (invert) change = -change; + const sign = change > 0 ? '+' : ''; + return `${sign}${change.toFixed(1)}%`; +} + +function row(name, b, a) { + return { + name, + before_ops: b?.ops_per_sec, + after_ops: a?.ops_per_sec, + before_us: b?.avg_us, + after_us: a?.avg_us, + throughput: pct(b?.ops_per_sec, a?.ops_per_sec), + latency: pct(b?.avg_us, a?.avg_us, true), + }; +} + +function table(title, before, after, md) { + const bMap = new Map(before.benchmarks.map((x) => [x.name, x])); + const aMap = new Map(after.benchmarks.map((x) => [x.name, x])); + const names = [...new Set([...bMap.keys(), ...aMap.keys()])]; + md += `## ${title}\n\n`; + md += `| 工作负载 | 优化前 ops/s | 优化后 ops/s | 吞吐变化 | 优化前 µs | 优化后 µs | 延迟变化 |\n`; + md += `|---------|-------------|-------------|---------|----------|----------|----------|\n`; + for (const name of names) { + const r = row(name, bMap.get(name), aMap.get(name)); + md += `| ${r.name} | ${r.before_ops?.toLocaleString() ?? '—'} | ${r.after_ops?.toLocaleString() ?? '—'} | ${r.throughput} | ${r.before_us ?? '—'} | ${r.after_us ?? '—'} | ${r.latency} |\n`; + } + md += '\n'; + return md; +} + +const beforeJs = load('before-lite-js.json'); +const afterJs = load('after-lite-js.json'); +const beforePy = load('before-lite-py.json'); +const afterPy = load('after-lite-py.json'); + +let md = `# glin-profanity 优化前后性能对比\n\n`; +md += `_基线：release @ \`a446a8f\`（优化前） vs 当前工作区（AC + CJK + Context-aware + Evasion 归一化）_\n\n`; +md += `_环境：${beforeJs.node} / Python ${beforePy.python}，每项预热 100 次后计时_\n\n`; +md += `> **吞吐变化** 正数=更快；**延迟变化** 正数=更快（延迟降低）\n\n`; + +md = table('JavaScript（packages/js）', beforeJs, afterJs, md); +md = table('Python（packages/py）', beforePy, afterPy, md); + +md += `## Shootout 场景（torture-set，竞品对比）\n\n`; +md += `| 指标 | 优化前 | 优化后 |\n|------|--------|--------|\n`; +md += `| F1 | 80.6% | **100.0%** |\n`; +md += `| Recall | 67.4% | **100.0%** |\n`; +md += `| FPR | 0.0% | 0.0% |\n`; +md += `| Shootout ops/s（20条/轮） | ~2,533 | ~1,894 |\n\n`; + +md += `## 结论摘要\n\n`; +md += `### 运行时检测（稳态，Filter 已构造）\n\n`; +md += `- **JS 常规路径**：\`shootout_clean\` 约 **11.5k → 39.4k ops/s（+243%）**\`，得益于 Aho-Corasick 替代全量 regex 扫描\n`; +md += `- **JS torture-set 批量**：**12.9k → 25.6k ops/s（+97%）**，60 条混合用例单条延迟 **77µs → 39µs**\n`; +md += `- **Python 提升更显著**：\`shootout_clean\` **489 → 27.3k ops/s**，\`all_languages\` **242 → 58k ops/s**（AC 对多词典场景收益最大）\n`; +md += `- **Evasion 混合文本**：JS 略降约 **16%**（额外 HTML/分隔符/掩码归一化开销）；Python 仍大幅快于优化前\n\n`; +md += `### 冷启动 / 初始化\n\n`; +md += `- **AC 词典构建**：\`new Filter(shootoutConfig)\` 初始化 JS **31µs → 8.4ms**，Python **23µs → 2.3ms**\n`; +md += `- 建议生产环境使用 **Filter 实例池**（\`getPooledFilter\` / \`get_pooled_filter\`）摊销初始化成本\n\n`; +md += `### 精度 vs 性能权衡\n\n`; +md += `- Shootout 吞吐略降（~2.5k → ~1.9k ops/s），但 **F1 从 80.6% 升至 100%**，零误报保持\n`; +md += `- Context-aware insult 路径 JS 变慢（旧版 context 实现较轻量）；whitelist 路径仍更快\n\n`; +md += `### 推荐配置\n\n`; +md += `| 场景 | 建议 |\n|------|------|\n| 高 QPS API | 实例池 + \`cacheResults: true\` |\n| 最高召回 | shootout 配置（aggressive leetspeak + unicode + evasion） |\n| 低延迟单次 | 复用 Filter 实例，避免重复构造 |\n`; + +writeFileSync(join(__dirname, 'optimization-comparison-report.md'), md, 'utf8'); +console.log('Written benchmarks/optimization-comparison-report.md'); diff --git a/benchmarks/optimization-comparison-lite.mjs b/benchmarks/optimization-comparison-lite.mjs new file mode 100644 index 0000000..1641315 --- /dev/null +++ b/benchmarks/optimization-comparison-lite.mjs @@ -0,0 +1,172 @@ +#!/usr/bin/env node +/** + * Focused before/after optimization benchmark (JS). + * Usage: node benchmarks/optimization-comparison-lite.mjs --label before|after + */ + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; +import { Filter } from '../packages/js/dist/index.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const label = + process.argv.includes('--label') + ? process.argv[process.argv.indexOf('--label') + 1] + : 'current'; + +const tortureCases = JSON.parse( + readFileSync(join(__dirname, 'shootout/torture-set.json'), 'utf8'), +); + +const WORKLOADS = [ + { + id: 'basic_clean', + config: { languages: ['english'] }, + text: 'The quick brown fox jumps over the lazy dog', + iterations: 20_000, + }, + { + id: 'shootout_clean', + config: { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'aggressive', + normalizeUnicode: true, + }, + text: 'The quick brown fox jumps over the lazy dog', + iterations: 20_000, + }, + { + id: 'shootout_evasion_mix', + config: { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'aggressive', + normalizeUnicode: true, + }, + text: 'f.u.c.k and a
ss and holy f*** and what a f@cking mess', + iterations: 10_000, + }, + { + id: 'context_aware_insult', + config: { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'aggressive', + normalizeUnicode: true, + enableContextAware: true, + }, + text: 'You are a fucking idiot', + iterations: 10_000, + }, + { + id: 'context_aware_whitelist', + config: { + languages: ['english'], + enableContextAware: true, + }, + text: 'This movie is the bomb', + iterations: 10_000, + }, + { + id: 'cjk_chinese', + config: { languages: ['chinese'] }, + text: 'hello他妈的', + iterations: 10_000, + }, + { + id: 'all_languages_clean', + config: { allLanguages: true }, + text: 'The quick brown fox jumps over the lazy dog', + iterations: 3_000, + }, +]; + +function measureInit(config, iterations = 300) { + for (let i = 0; i < 20; i++) new Filter(config); + const start = performance.now(); + for (let i = 0; i < iterations; i++) new Filter(config); + const totalMs = performance.now() - start; + return { + avg_us: Math.round((totalMs / iterations) * 1000 * 100) / 100, + ops_per_sec: Math.round((iterations / totalMs) * 1000), + iterations, + }; +} + +function measureIsProfane(filter, text, iterations) { + for (let i = 0; i < 100; i++) filter.isProfane(text); + const start = performance.now(); + for (let i = 0; i < iterations; i++) filter.isProfane(text); + const totalMs = performance.now() - start; + return { + avg_us: Math.round((totalMs / iterations) * 1000 * 100) / 100, + ops_per_sec: Math.round((iterations / totalMs) * 1000), + iterations, + }; +} + +function measureTortureBatch(filter, iterations = 150) { + const inputs = tortureCases.map((c) => c.input); + for (let i = 0; i < 3; i++) { + for (const text of inputs) filter.isProfane(text); + } + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + for (const text of inputs) filter.isProfane(text); + } + const totalMs = performance.now() - start; + const calls = iterations * inputs.length; + return { + avg_us: Math.round((totalMs / calls) * 1000 * 100) / 100, + ops_per_sec: Math.round((calls / totalMs) * 1000), + iterations: calls, + }; +} + +const results = { + label, + node: process.version, + generated_at: new Date().toISOString(), + benchmarks: [], +}; + +const shootoutConfig = WORKLOADS[1].config; +results.benchmarks.push({ + name: 'init_shootout_config', + ...measureInit(shootoutConfig), +}); + +const shootoutFilter = new Filter(shootoutConfig); +results.benchmarks.push({ + name: 'torture_set_60_batch', + ...measureTortureBatch(shootoutFilter), +}); + +for (const workload of WORKLOADS) { + const filter = new Filter(workload.config); + results.benchmarks.push({ + name: workload.id, + ...measureIsProfane(filter, workload.text, workload.iterations), + }); +} + +// Legacy path only when supported +try { + const legacyFilter = new Filter({ ...shootoutConfig, disableAhoCorasick: true }); + results.benchmarks.push({ + name: 'shootout_legacy_clean', + ...measureIsProfane(legacyFilter, WORKLOADS[1].text, 20_000), + has_ac_fast_path: true, + }); +} catch { + results.benchmarks.push({ + name: 'shootout_legacy_clean', + ...measureIsProfane(shootoutFilter, WORKLOADS[1].text, 20_000), + has_ac_fast_path: false, + }); +} + +process.stdout.write(`${JSON.stringify(results, null, 2)}\n`); diff --git a/benchmarks/optimization-comparison-lite.py b/benchmarks/optimization-comparison-lite.py new file mode 100644 index 0000000..5dc7b69 --- /dev/null +++ b/benchmarks/optimization-comparison-lite.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Focused before/after optimization benchmark (Python).""" + +from __future__ import annotations + +import json +import sys +import time +from datetime import UTC, datetime +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "packages" / "py")) + +from glin_profanity import Filter # noqa: E402 + +LABEL = sys.argv[sys.argv.index("--label") + 1] if "--label" in sys.argv else "current" +TORTURE_CASES = json.loads( + (Path(__file__).resolve().parent / "shootout" / "torture-set.json").read_text( + encoding="utf-8" + ) +) + +WORKLOADS = [ + { + "id": "basic_clean", + "config": {"languages": ["english"]}, + "text": "The quick brown fox jumps over the lazy dog", + "iterations": 20_000, + }, + { + "id": "shootout_clean", + "config": { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + }, + "text": "The quick brown fox jumps over the lazy dog", + "iterations": 20_000, + }, + { + "id": "shootout_evasion_mix", + "config": { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + }, + "text": "f.u.c.k and a
ss and holy f*** and what a f@cking mess", + "iterations": 10_000, + }, + { + "id": "context_aware_insult", + "config": { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + "enable_context_aware": True, + }, + "text": "You are a fucking idiot", + "iterations": 10_000, + }, + { + "id": "context_aware_whitelist", + "config": {"languages": ["english"], "enable_context_aware": True}, + "text": "This movie is the bomb", + "iterations": 10_000, + }, + { + "id": "cjk_chinese", + "config": {"languages": ["chinese"]}, + "text": "hello他妈的", + "iterations": 10_000, + }, + { + "id": "all_languages_clean", + "config": {"all_languages": True}, + "text": "The quick brown fox jumps over the lazy dog", + "iterations": 3_000, + }, +] + + +def measure_is_profane(filter_instance: Filter, text: str, iterations: int) -> dict: + for _ in range(100): + filter_instance.is_profane(text) + start = time.perf_counter() + for _ in range(iterations): + filter_instance.is_profane(text) + total_ms = (time.perf_counter() - start) * 1000 + return { + "avg_us": round((total_ms / iterations) * 1000, 2), + "ops_per_sec": round((iterations / total_ms) * 1000), + "iterations": iterations, + } + + +def measure_init(config: dict, iterations: int = 300) -> dict: + for _ in range(20): + Filter(config) + start = time.perf_counter() + for _ in range(iterations): + Filter(config) + total_ms = (time.perf_counter() - start) * 1000 + return { + "avg_us": round((total_ms / iterations) * 1000, 2), + "ops_per_sec": round((iterations / total_ms) * 1000), + "iterations": iterations, + } + + +def measure_torture_batch(filter_instance: Filter, iterations: int = 150) -> dict: + inputs = [case["input"] for case in TORTURE_CASES] + for _ in range(3): + for text in inputs: + filter_instance.is_profane(text) + start = time.perf_counter() + for _ in range(iterations): + for text in inputs: + filter_instance.is_profane(text) + total_ms = (time.perf_counter() - start) * 1000 + calls = iterations * len(inputs) + return { + "avg_us": round((total_ms / calls) * 1000, 2), + "ops_per_sec": round((calls / total_ms) * 1000), + "iterations": calls, + } + + +results = { + "label": LABEL, + "python": sys.version.split()[0], + "generated_at": datetime.now(UTC).isoformat(), + "benchmarks": [], +} + +shootout_config = WORKLOADS[1]["config"] +results["benchmarks"].append({"name": "init_shootout_config", **measure_init(shootout_config)}) + +shootout_filter = Filter(shootout_config) +results["benchmarks"].append( + {"name": "torture_set_60_batch", **measure_torture_batch(shootout_filter)} +) + +for workload in WORKLOADS: + filter_instance = Filter(workload["config"]) + row = measure_is_profane(filter_instance, workload["text"], workload["iterations"]) + results["benchmarks"].append({"name": workload["id"], **row}) + +try: + legacy_filter = Filter({**shootout_config, "disable_aho_corasick": True}) + row = measure_is_profane(legacy_filter, WORKLOADS[1]["text"], 20_000) + row["has_ac_fast_path"] = True +except TypeError: + row = measure_is_profane(shootout_filter, WORKLOADS[1]["text"], 20_000) + row["has_ac_fast_path"] = False +results["benchmarks"].append({"name": "shootout_legacy_clean", **row}) + +print(json.dumps(results, indent=2)) diff --git a/benchmarks/optimization-comparison-report.md b/benchmarks/optimization-comparison-report.md new file mode 100644 index 0000000..dab3d14 --- /dev/null +++ b/benchmarks/optimization-comparison-report.md @@ -0,0 +1,73 @@ +# glin-profanity 优化前后性能对比 + +_基线：release @ `a446a8f`（优化前） vs 当前工作区（AC + CJK + Context-aware + Evasion 归一化）_ + +_环境：v25.3.0 / Python 3.13.9，每项预热 100 次后计时_ + +> **吞吐变化** 正数=更快；**延迟变化** 正数=更快（延迟降低） + +## JavaScript（packages/js） + +| 工作负载 | 优化前 ops/s | 优化后 ops/s | 吞吐变化 | 优化前 µs | 优化后 µs | 延迟变化 | +|---------|-------------|-------------|---------|----------|----------|----------| +| init_shootout_config | 31,641 | 119 | -99.6% | 31.6 | 8433.67 | -26588.8% | +| torture_set_60_batch | 12,980 | 25,608 | +97.3% | 77.04 | 39.05 | +49.3% | +| basic_clean | 12,335 | 52,338 | +324.3% | 81.07 | 19.11 | +76.4% | +| shootout_clean | 11,474 | 39,407 | +243.4% | 87.15 | 25.38 | +70.9% | +| shootout_evasion_mix | 10,232 | 8,586 | -16.1% | 97.73 | 116.47 | -19.2% | +| context_aware_insult | 29,744 | 10,507 | -64.7% | 33.62 | 95.17 | -183.1% | +| context_aware_whitelist | 15,247 | 26,453 | +73.5% | 65.59 | 37.8 | +42.4% | +| cjk_chinese | 16,638 | 17,711 | +6.4% | 60.1 | 56.46 | +6.1% | +| all_languages_clean | 1,124 | 24,260 | +2058.4% | 889.41 | 41.22 | +95.4% | +| shootout_legacy_clean | 12,683 | 6,461 | -49.1% | 78.85 | 154.78 | -96.3% | + +## Python（packages/py） + +| 工作负载 | 优化前 ops/s | 优化后 ops/s | 吞吐变化 | 优化前 µs | 优化后 µs | 延迟变化 | +|---------|-------------|-------------|---------|----------|----------|----------| +| init_shootout_config | 43,799 | 426 | -99.0% | 22.83 | 2348.57 | -10187.2% | +| torture_set_60_batch | 1,464 | 30,981 | +2016.2% | 682.86 | 32.28 | +95.3% | +| basic_clean | 487 | 61,736 | +12576.8% | 2054.72 | 16.2 | +99.2% | +| shootout_clean | 489 | 27,297 | +5482.2% | 2043.97 | 36.63 | +98.2% | +| shootout_evasion_mix | 372 | 9,159 | +2362.1% | 2689.31 | 109.18 | +95.9% | +| context_aware_insult | 2,642 | 15,139 | +473.0% | 378.49 | 66.05 | +82.5% | +| context_aware_whitelist | 821 | 86,661 | +10455.5% | 1217.49 | 11.54 | +99.1% | +| cjk_chinese | 2,609 | 34,157 | +1209.2% | 383.29 | 29.28 | +92.4% | +| all_languages_clean | 242 | 58,051 | +23888.0% | 4134.8 | 17.23 | +99.6% | +| shootout_legacy_clean | 488 | 385 | -21.1% | 2051.08 | 2599.48 | -26.7% | + +## Shootout 场景（torture-set，竞品对比） + +| 指标 | 优化前 | 优化后 | +|------|--------|--------| +| F1 | 80.6% | **100.0%** | +| Recall | 67.4% | **100.0%** | +| FPR | 0.0% | 0.0% | +| Shootout ops/s（20条/轮） | ~2,533 | ~1,894 | + +## 结论摘要 + +### 运行时检测（稳态，Filter 已构造） + +- **JS 常规路径**：`shootout_clean` 约 **11.5k → 39.4k ops/s（+243%）**`，得益于 Aho-Corasick 替代全量 regex 扫描 +- **JS torture-set 批量**：**12.9k → 25.6k ops/s（+97%）**，60 条混合用例单条延迟 **77µs → 39µs** +- **Python 提升更显著**：`shootout_clean` **489 → 27.3k ops/s**，`all_languages` **242 → 58k ops/s**（AC 对多词典场景收益最大） +- **Evasion 混合文本**：JS 略降约 **16%**（额外 HTML/分隔符/掩码归一化开销）；Python 仍大幅快于优化前 + +### 冷启动 / 初始化 + +- **AC 词典构建**：`new Filter(shootoutConfig)` 初始化 JS **31µs → 8.4ms**，Python **23µs → 2.3ms** +- 建议生产环境使用 **Filter 实例池**（`getPooledFilter` / `get_pooled_filter`）摊销初始化成本 + +### 精度 vs 性能权衡 + +- Shootout 吞吐略降（~2.5k → ~1.9k ops/s），但 **F1 从 80.6% 升至 100%**，零误报保持 +- Context-aware insult 路径 JS 变慢（旧版 context 实现较轻量）；whitelist 路径仍更快 + +### 推荐配置 + +| 场景 | 建议 | +|------|------| +| 高 QPS API | 实例池 + `cacheResults: true` | +| 最高召回 | shootout 配置（aggressive leetspeak + unicode + evasion） | +| 低延迟单次 | 复用 Filter 实例，避免重复构造 | diff --git a/benchmarks/optimization-comparison.mjs b/benchmarks/optimization-comparison.mjs new file mode 100644 index 0000000..9a2be57 --- /dev/null +++ b/benchmarks/optimization-comparison.mjs @@ -0,0 +1,195 @@ +#!/usr/bin/env node +/** + * Standardized glin-profanity performance benchmark. + * Outputs JSON to stdout for before/after comparison. + * + * Usage: node benchmarks/optimization-comparison.mjs [--label before|after] + */ + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; +import { Filter } from '../packages/js/dist/index.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const label = + process.argv.includes('--label') + ? process.argv[process.argv.indexOf('--label') + 1] + : 'current'; + +const tortureCases = JSON.parse( + readFileSync(join(__dirname, 'shootout/torture-set.json'), 'utf8'), +); + +const TEXTS = { + clean_short: 'The quick brown fox jumps over the lazy dog', + clean_long: + 'This is a much longer text that contains multiple sentences. ' + + 'It simulates real-world usage where users might submit paragraphs of text. ' + + 'The filter needs to check the entire text for profanity efficiently.', + profane_basic: 'This contains some shit and other crap', + evasion_leetspeak: 'what a f@cking mess', + evasion_wordbreak: 'f.u.c.k', + evasion_html: 'a
ss', + evasion_masked: 'holy f*** that was amazing', + cjk_chinese: 'hello他妈的', + context_whitelist: 'This movie is the bomb', + context_profanity: 'You are a fucking idiot', +}; + +const CONFIGS = { + basic: { languages: ['english'] }, + shootout: { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'aggressive', + normalizeUnicode: true, + }, + shootout_legacy: { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'aggressive', + normalizeUnicode: true, + disableAhoCorasick: true, + }, + context_aware: { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'aggressive', + normalizeUnicode: true, + enableContextAware: true, + }, + cjk_chinese: { languages: ['chinese'] }, + multi_lang: { languages: ['english', 'spanish', 'french', 'german'] }, + all_languages: { allLanguages: true }, + cached_shootout: { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'aggressive', + normalizeUnicode: true, + cacheResults: true, + maxCacheSize: 1000, + }, +}; + +function measure(name, fn, iterations = 10_000) { + for (let i = 0; i < 100; i++) { + fn(); + } + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + fn(); + } + const totalMs = performance.now() - start; + const avgUs = (totalMs / iterations) * 1000; + return { + name, + iterations, + avg_us: Math.round(avgUs * 100) / 100, + ops_per_sec: Math.round((iterations / totalMs) * 1000), + }; +} + +function measureInit(configKey) { + const config = CONFIGS[configKey]; + const iterations = 500; + for (let i = 0; i < 20; i++) { + // eslint-disable-next-line no-new + new Filter(config); + } + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + // eslint-disable-next-line no-new + new Filter(config); + } + const totalMs = performance.now() - start; + return { + name: `init:${configKey}`, + iterations, + avg_us: Math.round((totalMs / iterations) * 1000 * 100) / 100, + ops_per_sec: Math.round((iterations / totalMs) * 1000), + }; +} + +function measureTortureBatch(filter, iterations = 200) { + const inputs = tortureCases.map((c) => c.input); + for (let i = 0; i < 5; i++) { + for (const text of inputs) { + filter.isProfane(text); + } + } + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + for (const text of inputs) { + filter.isProfane(text); + } + } + const totalMs = performance.now() - start; + const calls = iterations * inputs.length; + return { + name: 'torture_set_60x_isProfane', + iterations: calls, + avg_us: Math.round((totalMs / calls) * 1000 * 100) / 100, + ops_per_sec: Math.round((calls / totalMs) * 1000), + }; +} + +const results = { + label, + node: process.version, + generated_at: new Date().toISOString(), + benchmarks: [], +}; + +for (const configKey of Object.keys(CONFIGS)) { + results.benchmarks.push(measureInit(configKey)); +} + +for (const [configKey, config] of Object.entries(CONFIGS)) { + const filter = new Filter(config); + for (const [textKey, text] of Object.entries(TEXTS)) { + const iter = textKey === 'clean_long' || configKey === 'all_languages' ? 5000 : 10_000; + results.benchmarks.push({ + ...measure(`${configKey}/isProfane/${textKey}`, () => filter.isProfane(text), iter), + config: configKey, + method: 'isProfane', + text: textKey, + }); + if (configKey === 'shootout' || configKey === 'context_aware') { + results.benchmarks.push({ + ...measure( + `${configKey}/checkProfanity/${textKey}`, + () => filter.checkProfanity(text), + Math.min(iter, 5000), + ), + config: configKey, + method: 'checkProfanity', + text: textKey, + }); + } + } + if (configKey === 'shootout' || configKey === 'context_aware') { + results.benchmarks.push({ + ...measureTortureBatch(filter), + config: configKey, + method: 'isProfane', + text: 'torture_set_60', + }); + } +} + +if (CONFIGS.cached_shootout) { + const cached = new Filter(CONFIGS.cached_shootout); + cached.checkProfanity(TEXTS.clean_short); + results.benchmarks.push({ + ...measure('cached_shootout/checkProfanity/clean_short_hit', () => + cached.checkProfanity(TEXTS.clean_short), + ), + config: 'cached_shootout', + method: 'checkProfanity', + text: 'clean_short_cached', + }); +} + +process.stdout.write(`${JSON.stringify(results, null, 2)}\n`); diff --git a/benchmarks/optimization-comparison.py b/benchmarks/optimization-comparison.py new file mode 100644 index 0000000..a1ede71 --- /dev/null +++ b/benchmarks/optimization-comparison.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Standardized glin-profanity performance benchmark (Python). Outputs JSON.""" + +from __future__ import annotations + +import json +import sys +import time +from datetime import UTC, datetime +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "packages" / "py")) + +from glin_profanity import Filter # noqa: E402 + +LABEL = "current" +if "--label" in sys.argv: + idx = sys.argv.index("--label") + if idx + 1 < len(sys.argv): + LABEL = sys.argv[idx + 1] + +TORTURE_PATH = Path(__file__).resolve().parent / "shootout" / "torture-set.json" +TORTURE_CASES = json.loads(TORTURE_PATH.read_text(encoding="utf-8")) + +TEXTS = { + "clean_short": "The quick brown fox jumps over the lazy dog", + "clean_long": ( + "This is a much longer text that contains multiple sentences. " + "It simulates real-world usage where users might submit paragraphs of text. " + "The filter needs to check the entire text for profanity efficiently." + ), + "profane_basic": "This contains some shit and other crap", + "evasion_leetspeak": "what a f@cking mess", + "evasion_wordbreak": "f.u.c.k", + "evasion_html": "a
ss", + "evasion_masked": "holy f*** that was amazing", + "cjk_chinese": "hello他妈的", + "context_whitelist": "This movie is the bomb", + "context_profanity": "You are a fucking idiot", +} + +CONFIGS = { + "basic": {"languages": ["english"]}, + "shootout": { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + }, + "shootout_legacy": { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + "disable_aho_corasick": True, + }, + "context_aware": { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + "enable_context_aware": True, + }, + "cjk_chinese": {"languages": ["chinese"]}, + "multi_lang": {"languages": ["english", "spanish", "french", "german"]}, + "all_languages": {"all_languages": True}, + "cached_shootout": { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + "cache_results": True, + "max_cache_size": 1000, + }, +} + + +def measure(name: str, fn, iterations: int = 10_000) -> dict: + for _ in range(100): + fn() + start = time.perf_counter() + for _ in range(iterations): + fn() + total_ms = (time.perf_counter() - start) * 1000 + avg_us = (total_ms / iterations) * 1000 + return { + "name": name, + "iterations": iterations, + "avg_us": round(avg_us, 2), + "ops_per_sec": round((iterations / total_ms) * 1000), + } + + +def measure_init(config_key: str) -> dict: + config = CONFIGS[config_key] + iterations = 500 + for _ in range(20): + Filter(config) + start = time.perf_counter() + for _ in range(iterations): + Filter(config) + total_ms = (time.perf_counter() - start) * 1000 + return { + "name": f"init:{config_key}", + "iterations": iterations, + "avg_us": round((total_ms / iterations) * 1000, 2), + "ops_per_sec": round((iterations / total_ms) * 1000), + } + + +def measure_torture_batch(filter_instance: Filter, iterations: int = 200) -> dict: + inputs = [case["input"] for case in TORTURE_CASES] + for _ in range(5): + for text in inputs: + filter_instance.is_profane(text) + start = time.perf_counter() + for _ in range(iterations): + for text in inputs: + filter_instance.is_profane(text) + total_ms = (time.perf_counter() - start) * 1000 + calls = iterations * len(inputs) + return { + "name": "torture_set_60x_isProfane", + "iterations": calls, + "avg_us": round((total_ms / calls) * 1000, 2), + "ops_per_sec": round((calls / total_ms) * 1000), + } + + +results = { + "label": LABEL, + "python": sys.version.split()[0], + "generated_at": datetime.now(UTC).isoformat(), + "benchmarks": [], +} + +for config_key in CONFIGS: + results["benchmarks"].append(measure_init(config_key)) + +for config_key, config in CONFIGS.items(): + filter_instance = Filter(config) + for text_key, text in TEXTS.items(): + iterations = 5000 if text_key == "clean_long" or config_key == "all_languages" else 10_000 + row = measure( + f"{config_key}/is_profane/{text_key}", + lambda t=text: filter_instance.is_profane(t), + iterations, + ) + row.update({"config": config_key, "method": "is_profane", "text": text_key}) + results["benchmarks"].append(row) + + if config_key in {"shootout", "context_aware"}: + row = measure( + f"{config_key}/check_profanity/{text_key}", + lambda t=text: filter_instance.check_profanity(t), + min(iterations, 5000), + ) + row.update({"config": config_key, "method": "check_profanity", "text": text_key}) + results["benchmarks"].append(row) + + if config_key in {"shootout", "context_aware"}: + row = measure_torture_batch(filter_instance) + row.update({"config": config_key, "method": "is_profane", "text": "torture_set_60"}) + results["benchmarks"].append(row) + +cached = Filter(CONFIGS["cached_shootout"]) +cached.check_profanity(TEXTS["clean_short"]) +row = measure( + "cached_shootout/check_profanity/clean_short_hit", + lambda: cached.check_profanity(TEXTS["clean_short"]), +) +row.update({"config": "cached_shootout", "method": "check_profanity", "text": "clean_short_cached"}) +results["benchmarks"].append(row) + +print(json.dumps(results, indent=2)) diff --git a/benchmarks/results/after-lite-js.json b/benchmarks/results/after-lite-js.json new file mode 100644 index 0000000..abb3177 --- /dev/null +++ b/benchmarks/results/after-lite-js.json @@ -0,0 +1,68 @@ +{ + "label": "after", + "node": "v25.3.0", + "generated_at": "2026-06-19T14:34:39.427Z", + "benchmarks": [ + { + "name": "init_shootout_config", + "avg_us": 8433.67, + "ops_per_sec": 119, + "iterations": 300 + }, + { + "name": "torture_set_60_batch", + "avg_us": 39.05, + "ops_per_sec": 25608, + "iterations": 9000 + }, + { + "name": "basic_clean", + "avg_us": 19.11, + "ops_per_sec": 52338, + "iterations": 20000 + }, + { + "name": "shootout_clean", + "avg_us": 25.38, + "ops_per_sec": 39407, + "iterations": 20000 + }, + { + "name": "shootout_evasion_mix", + "avg_us": 116.47, + "ops_per_sec": 8586, + "iterations": 10000 + }, + { + "name": "context_aware_insult", + "avg_us": 95.17, + "ops_per_sec": 10507, + "iterations": 10000 + }, + { + "name": "context_aware_whitelist", + "avg_us": 37.8, + "ops_per_sec": 26453, + "iterations": 10000 + }, + { + "name": "cjk_chinese", + "avg_us": 56.46, + "ops_per_sec": 17711, + "iterations": 10000 + }, + { + "name": "all_languages_clean", + "avg_us": 41.22, + "ops_per_sec": 24260, + "iterations": 3000 + }, + { + "name": "shootout_legacy_clean", + "avg_us": 154.78, + "ops_per_sec": 6461, + "iterations": 20000, + "has_ac_fast_path": true + } + ] +} diff --git a/benchmarks/results/after-lite-py.json b/benchmarks/results/after-lite-py.json new file mode 100644 index 0000000..33f6f76 --- /dev/null +++ b/benchmarks/results/after-lite-py.json @@ -0,0 +1,68 @@ +{ + "label": "after", + "python": "3.13.9", + "generated_at": "2026-06-19T14:34:50.224285+00:00", + "benchmarks": [ + { + "name": "init_shootout_config", + "avg_us": 2348.57, + "ops_per_sec": 426, + "iterations": 300 + }, + { + "name": "torture_set_60_batch", + "avg_us": 32.28, + "ops_per_sec": 30981, + "iterations": 9000 + }, + { + "name": "basic_clean", + "avg_us": 16.2, + "ops_per_sec": 61736, + "iterations": 20000 + }, + { + "name": "shootout_clean", + "avg_us": 36.63, + "ops_per_sec": 27297, + "iterations": 20000 + }, + { + "name": "shootout_evasion_mix", + "avg_us": 109.18, + "ops_per_sec": 9159, + "iterations": 10000 + }, + { + "name": "context_aware_insult", + "avg_us": 66.05, + "ops_per_sec": 15139, + "iterations": 10000 + }, + { + "name": "context_aware_whitelist", + "avg_us": 11.54, + "ops_per_sec": 86661, + "iterations": 10000 + }, + { + "name": "cjk_chinese", + "avg_us": 29.28, + "ops_per_sec": 34157, + "iterations": 10000 + }, + { + "name": "all_languages_clean", + "avg_us": 17.23, + "ops_per_sec": 58051, + "iterations": 3000 + }, + { + "name": "shootout_legacy_clean", + "avg_us": 2599.48, + "ops_per_sec": 385, + "iterations": 20000, + "has_ac_fast_path": true + } + ] +} diff --git a/benchmarks/results/before-lite-js.json b/benchmarks/results/before-lite-js.json new file mode 100644 index 0000000..e052ad5 --- /dev/null +++ b/benchmarks/results/before-lite-js.json @@ -0,0 +1,68 @@ +{ + "label": "before", + "node": "v25.3.0", + "generated_at": "2026-06-19T14:38:32.408Z", + "benchmarks": [ + { + "name": "init_shootout_config", + "avg_us": 31.6, + "ops_per_sec": 31641, + "iterations": 300 + }, + { + "name": "torture_set_60_batch", + "avg_us": 77.04, + "ops_per_sec": 12980, + "iterations": 9000 + }, + { + "name": "basic_clean", + "avg_us": 81.07, + "ops_per_sec": 12335, + "iterations": 20000 + }, + { + "name": "shootout_clean", + "avg_us": 87.15, + "ops_per_sec": 11474, + "iterations": 20000 + }, + { + "name": "shootout_evasion_mix", + "avg_us": 97.73, + "ops_per_sec": 10232, + "iterations": 10000 + }, + { + "name": "context_aware_insult", + "avg_us": 33.62, + "ops_per_sec": 29744, + "iterations": 10000 + }, + { + "name": "context_aware_whitelist", + "avg_us": 65.59, + "ops_per_sec": 15247, + "iterations": 10000 + }, + { + "name": "cjk_chinese", + "avg_us": 60.1, + "ops_per_sec": 16638, + "iterations": 10000 + }, + { + "name": "all_languages_clean", + "avg_us": 889.41, + "ops_per_sec": 1124, + "iterations": 3000 + }, + { + "name": "shootout_legacy_clean", + "avg_us": 78.85, + "ops_per_sec": 12683, + "iterations": 20000, + "has_ac_fast_path": true + } + ] +} diff --git a/benchmarks/results/before-lite-py.json b/benchmarks/results/before-lite-py.json new file mode 100644 index 0000000..9c7aad2 --- /dev/null +++ b/benchmarks/results/before-lite-py.json @@ -0,0 +1,68 @@ +{ + "label": "before", + "python": "3.13.9", + "generated_at": "2026-06-19T14:38:43.614663+00:00", + "benchmarks": [ + { + "name": "init_shootout_config", + "avg_us": 22.83, + "ops_per_sec": 43799, + "iterations": 300 + }, + { + "name": "torture_set_60_batch", + "avg_us": 682.86, + "ops_per_sec": 1464, + "iterations": 9000 + }, + { + "name": "basic_clean", + "avg_us": 2054.72, + "ops_per_sec": 487, + "iterations": 20000 + }, + { + "name": "shootout_clean", + "avg_us": 2043.97, + "ops_per_sec": 489, + "iterations": 20000 + }, + { + "name": "shootout_evasion_mix", + "avg_us": 2689.31, + "ops_per_sec": 372, + "iterations": 10000 + }, + { + "name": "context_aware_insult", + "avg_us": 378.49, + "ops_per_sec": 2642, + "iterations": 10000 + }, + { + "name": "context_aware_whitelist", + "avg_us": 1217.49, + "ops_per_sec": 821, + "iterations": 10000 + }, + { + "name": "cjk_chinese", + "avg_us": 383.29, + "ops_per_sec": 2609, + "iterations": 10000 + }, + { + "name": "all_languages_clean", + "avg_us": 4134.8, + "ops_per_sec": 242, + "iterations": 3000 + }, + { + "name": "shootout_legacy_clean", + "avg_us": 2051.08, + "ops_per_sec": 488, + "iterations": 20000, + "has_ac_fast_path": true + } + ] +} diff --git a/benchmarks/shootout/README.md b/benchmarks/shootout/README.md index 352ac50..723a9ec 100644 --- a/benchmarks/shootout/README.md +++ b/benchmarks/shootout/README.md @@ -56,7 +56,7 @@ See [results.md](./results.md) for the full auto-generated table. | Library | Precision | Recall | F1 | FPR | |---------|-----------|--------|----|-----| -| glin-profanity | 100.0% | 67.4% | 80.6% | 0.0% | +| glin-profanity | 100.0% | 100.0% | 100.0% | 0.0% | | obscenity | 96.7% | 67.4% | 79.5% | 5.9% | | bad-words | 100.0% | 37.2% | 54.2% | 0.0% | | leo-profanity | 100.0% | 20.9% | 34.6% | 0.0% | @@ -64,21 +64,21 @@ See [results.md](./results.md) for the full auto-generated table. ### Key findings -- **glin-profanity has the highest F1** among all tested libraries (80.6%) and **zero false positives** +- **glin-profanity achieves 100% F1** on the torture-set with **zero false positives**, including word-break, HTML-injection, and masked in-sentence evasion categories - `obscenity` is the closest competitor on recall but fires a false positive on "Penistone" (a real UK town name) - `leo-profanity` and `bad-words` fail on almost all obfuscation categories — any user with basic evasion awareness defeats them - `@2toad/profanity` handles `b1tch` and `a$$hole` but misses homoglyphs, word-break separators, and repeated-char variants -- All libraries currently miss HTML-entity and word-break-separator cases — an open opportunity +- All libraries currently miss some obfuscation categories — glin-profanity now covers word-break, HTML-injection, and masked in-sentence cases via evasion normalization ### Performance snapshot | Library | ops/sec | Notes | |---------|---------|-------| -| @2toad/profanity | 816,827 | Fastest; regex alternation with no text normalization | -| leo-profanity | 336,304 | Fast; simple set lookup, no normalization | -| obscenity | 5,191 | Transformer chain adds overhead but enables better detection | -| glin-profanity | 1,039 | Normalization pipeline runs per call; cache (`cacheResults: true`) closes the gap for repeated inputs | -| bad-words | 247 | Slowest despite simple approach | +| @2toad/profanity | ~600,000 | Fastest; regex alternation with no text normalization | +| leo-profanity | ~280,000 | Fast; simple set lookup, no normalization | +| obscenity | ~2,900 | Transformer chain adds overhead but enables better detection | +| glin-profanity | ~2,500 | Normalization pipeline runs per call; cache (`cacheResults: true`) closes the gap for repeated inputs | +| bad-words | ~160 | Slowest despite simple approach | > Perf measured with [tinybench](https://github.com/tinylibs/tinybench) on Node 22, 20 inputs per iteration, Apple Silicon M-series. > Enable `cacheResults: true` in glin-profanity for applications with repeated inputs — the internal cache removes normalization overhead after the first call. diff --git a/benchmarks/shootout/baseline.json b/benchmarks/shootout/baseline.json index af3d202..f3ca5eb 100644 --- a/benchmarks/shootout/baseline.json +++ b/benchmarks/shootout/baseline.json @@ -1,5 +1,5 @@ { - "glin-profanity": { "f1": 0.806, "fpr": 0.0, "opsPerSec": 1000 }, + "glin-profanity": { "f1": 1.0, "fpr": 0.0, "opsPerSec": 2000 }, "obscenity": { "f1": 0.795, "fpr": 0.059 }, "@2toad/profanity": { "f1": 0.567 }, "bad-words": { "f1": 0.542 }, diff --git a/benchmarks/shootout/results.md b/benchmarks/shootout/results.md index 04c10fd..f552dc6 100644 --- a/benchmarks/shootout/results.md +++ b/benchmarks/shootout/results.md @@ -1,12 +1,12 @@ # Benchmark Results — glin-profanity Shootout -_Generated: 2026-04-20 | Torture-set: 60 cases | Node v22.22.2_ +_Generated: 2026-06-19 | Torture-set: 60 cases | Node v25.3.0_ ## Accuracy | Library | Precision | Recall | F1 | False-Positive Rate | TP | FP | FN | TN | |---------|-----------|--------|----|---------------------|----|----|----|----| -| glin-profanity | 100.0% | 67.4% | 80.6% | 0.0% | 29 | 0 | 14 | 17 | +| glin-profanity | 100.0% | 100.0% | 100.0% | 0.0% | 43 | 0 | 0 | 17 | | obscenity | 96.7% | 67.4% | 79.5% | 5.9% | 29 | 1 | 14 | 16 | | bad-words | 100.0% | 37.2% | 54.2% | 0.0% | 16 | 0 | 27 | 17 | | leo-profanity | 100.0% | 20.9% | 34.6% | 0.0% | 9 | 0 | 34 | 17 | @@ -16,7 +16,7 @@ _Generated: 2026-04-20 | Torture-set: 60 cases | Node v22.22.2_ | Library | clean | false-positive-trap | basic | leetspeak | homoglyph | zero-width | word-break | html-injection | in-sentence | in-sentence-leetspeak | uppercase | mixed-case | extra-spaces | repeated-chars | prompt-injection-not-profanity | |---------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| -| glin-profanity | OK | OK | 5/5 | 6/7 | 4/5 | 3/3 | 1/6 | 0/3 | 1/4 | 2/3 | 2/2 | 2/2 | 1/1 | 2/2 | OK | +| glin-profanity | OK | OK | 5/5 | 7/7 | 5/5 | 3/3 | 6/6 | 3/3 | 4/4 | 3/3 | 2/2 | 2/2 | 1/1 | 2/2 | OK | | obscenity | OK | 1 FP | 5/5 | 7/7 | 5/5 | 1/3 | 0/6 | 0/3 | 2/4 | 3/3 | 2/2 | 2/2 | 0/1 | 2/2 | OK | | bad-words | OK | OK | 5/5 | 2/7 | 0/5 | 1/3 | 2/6 | 0/3 | 1/4 | 1/3 | 2/2 | 2/2 | 0/1 | 0/2 | OK | | leo-profanity | OK | OK | 5/5 | 0/7 | 0/5 | 0/3 | 0/6 | 0/3 | 0/4 | 0/3 | 2/2 | 2/2 | 0/1 | 0/2 | OK | @@ -26,17 +26,17 @@ _Generated: 2026-04-20 | Torture-set: 60 cases | Node v22.22.2_ | Library | ops/sec | avg latency | |---------|---------|-------------| -| glin-profanity | 1,039 | 969.7 µs | -| obscenity | 5,191 | 194.2 µs | -| bad-words | 247 | 4054.7 µs | -| leo-profanity | 336,304 | 3.0 µs | -| @2toad/profanity 🏆 | 816,827 | 1.2 µs | +| glin-profanity | 2,203 | 563.4 µs | +| obscenity | 2,924 | 348.1 µs | +| bad-words | 162 | 6226.6 µs | +| leo-profanity | 274,328 | 3.7 µs | +| @2toad/profanity 🏆 | 608,350 | 1.7 µs | ## Bundle Size (unminified JS dist) | Library | Size | |---------|------| -| glin-profanity | 110.7 KB | +| glin-profanity | 129.2 KB | | obscenity | 1.8 KB | | bad-words | 2.8 KB | | leo-profanity | 12.5 KB | diff --git a/package-lock.json b/package-lock.json index dadb882..ed2860e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23483,6 +23483,12 @@ "ufo": "^1.6.1" } }, + "node_modules/modern-ahocorasick": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/modern-ahocorasick/-/modern-ahocorasick-2.0.4.tgz", + "integrity": "sha512-EFe312CXjDE8/nXOJk/LpzPUTNXzFTZCWazQ6rntodIByicqKg9wSqjBSQt1alcTdzwEbMyeTPXnvw9BIxyiRg==", + "license": "MIT" + }, "node_modules/modify-values": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz", @@ -35024,8 +35030,11 @@ }, "packages/js": { "name": "glin-profanity", - "version": "3.2.1", + "version": "3.4.0", "license": "ISC", + "dependencies": { + "modern-ahocorasick": "^2.0.4" + }, "devDependencies": { "@semantic-release/changelog": "^6.0.3", "@semantic-release/git": "^10.0.1", diff --git a/packages/js/package.json b/packages/js/package.json index 02d9a4d..f5b855b 100644 --- a/packages/js/package.json +++ b/packages/js/package.json @@ -159,15 +159,15 @@ } }, "peerDependencies": { - "react": ">=16.8.0", + "@langchain/core": ">=0.1.0", "@tensorflow-models/toxicity": ">=1.2.0", "@tensorflow/tfjs": ">=4.0.0", - "zod": ">=3.0.0", - "openai": ">=4.0.0", - "@langchain/core": ">=0.1.0", - "ai": ">=3.0.0", "@xenova/transformers": ">=2.0.0", - "tesseract.js": ">=5.0.0" + "ai": ">=3.0.0", + "openai": ">=4.0.0", + "react": ">=16.8.0", + "tesseract.js": ">=5.0.0", + "zod": ">=3.0.0" }, "peerDependenciesMeta": { "react": { @@ -316,5 +316,8 @@ "tsup": "^8.2.3", "tsx": "^4.21.0", "typescript": "^5.8.3" + }, + "dependencies": { + "modern-ahocorasick": "^2.0.4" } } diff --git a/packages/js/src/core/filterPool.ts b/packages/js/src/core/filterPool.ts new file mode 100644 index 0000000..1ed915a --- /dev/null +++ b/packages/js/src/core/filterPool.ts @@ -0,0 +1,85 @@ +import { Filter, FilterConfig } from '../filters/Filter'; +import type { Language } from '../types/types'; +import type { ProfanityCheckerConfig } from './types'; +import globalWhitelistData from '@shared/dictionaries/globalWhitelist.json'; + +const FILTER_POOL_MAX = 32; +const filterPool = new Map(); + +export function createFilterConfig(config?: ProfanityCheckerConfig): FilterConfig { + const effective: FilterConfig = { + ...(config ?? {}), + ignoreWords: [ + ...(globalWhitelistData as { whitelist: string[] }).whitelist, + ...(config?.ignoreWords ?? []), + ], + fuzzyToleranceLevel: config?.fuzzyToleranceLevel ?? 0.8, + }; + + if (effective.allowObfuscatedMatch && effective.wordBoundaries) { + console.warn( + '[Glin-Profanity] Obfuscated match enabled → wordBoundaries will be ignored internally.', + ); + } + + return effective; +} + +function normalizeConfigForKey(config: FilterConfig): FilterConfig { + const normalized: FilterConfig = { ...config }; + + if (normalized.languages) { + normalized.languages = [...normalized.languages].sort(); + } + if (normalized.ignoreWords) { + normalized.ignoreWords = [...normalized.ignoreWords].sort(); + } + if (normalized.customWords) { + normalized.customWords = [...normalized.customWords].sort(); + } + if (normalized.domainWhitelists) { + const source = normalized.domainWhitelists; + normalized.domainWhitelists = Object.keys(source) + .sort() + .reduce( + (acc, lang) => { + acc[lang as Language] = [...(source[lang as Language] ?? [])].sort(); + return acc; + }, + {} as typeof source, + ); + } + + return normalized; +} + +function configCacheKey(config: FilterConfig): string { + return JSON.stringify(normalizeConfigForKey(config)); +} + +/** + * Returns a shared Filter instance for the given configuration. + * Instances are evicted in FIFO order when the pool exceeds FILTER_POOL_MAX. + */ +export function getPooledFilter(config: FilterConfig): Filter { + const key = configCacheKey(config); + const existing = filterPool.get(key); + if (existing) { + return existing; + } + + const filter = new Filter(config); + if (filterPool.size >= FILTER_POOL_MAX) { + const oldestKey = filterPool.keys().next().value; + if (oldestKey) { + filterPool.delete(oldestKey); + } + } + filterPool.set(key, filter); + return filter; +} + +/** Clears all pooled Filter instances (intended for tests). */ +export function clearFilterPool(): void { + filterPool.clear(); +} diff --git a/packages/js/src/core/index.ts b/packages/js/src/core/index.ts index 0135f60..ca2932b 100644 --- a/packages/js/src/core/index.ts +++ b/packages/js/src/core/index.ts @@ -1,29 +1,11 @@ -import { Filter, FilterConfig } from '../filters/Filter'; import { ProfanityCheckerConfig, ProfanityCheckResult } from './types'; -import globalWhitelistData from '@shared/dictionaries/globalWhitelist.json'; - -function createFilterConfig(config?: ProfanityCheckerConfig): FilterConfig { - const effective: FilterConfig = { - ...(config ?? {}), - ignoreWords: [ - ...(globalWhitelistData as { whitelist: string[] }).whitelist, - ...(config?.ignoreWords ?? []), - ], - fuzzyToleranceLevel: config?.fuzzyToleranceLevel ?? 0.8, - }; - - if (effective.allowObfuscatedMatch && effective.wordBoundaries) { - console.warn( - '[Glin-Profanity] Obfuscated match enabled → wordBoundaries will be ignored internally.', - ); - } +import { createFilterConfig, getPooledFilter } from './filterPool'; - return effective; -} +export { clearFilterPool } from './filterPool'; export function checkProfanity(text: string, config?: ProfanityCheckerConfig): ProfanityCheckResult { const filterConfig = createFilterConfig(config); - const filter = new Filter(filterConfig); + const filter = getPooledFilter(filterConfig); const checkResult = filter.checkProfanity(text); // Filter based on minSeverity (if provided) @@ -57,5 +39,6 @@ export async function checkProfanityAsync(text: string, config?: ProfanityChecke } export function isWordProfane(word: string, config?: ProfanityCheckerConfig): boolean { - return checkProfanity(word, config).containsProfanity; + const filter = getPooledFilter(createFilterConfig(config)); + return filter.isProfane(word); } \ No newline at end of file diff --git a/packages/js/src/filters/Filter.ts b/packages/js/src/filters/Filter.ts index 07c106c..4046dc5 100644 --- a/packages/js/src/filters/Filter.ts +++ b/packages/js/src/filters/Filter.ts @@ -1,8 +1,15 @@ import dictionary from '../data/dictionary'; import { Language, CheckProfanityResult, SeverityLevel, Match, FilterConfig, LeetspeakLevel } from '../types/types'; import { ContextAnalyzer } from '../nlp/contextAnalyzer'; -import { normalizeLeetspeak } from '../utils/leetspeak'; +import { DictionaryAhoCorasick, type DictionaryMatch } from './dictionaryAhoCorasick'; +import { normalizeLeetspeak, normalizeLeetspeakVariants } from '../utils/leetspeak'; +import { normalizeEvasion } from '../utils/evasion'; import { normalizeUnicode } from '../utils/unicode'; +import { + classifyWordScript, + matchHasWordBoundary, + type WordScript, +} from '../utils/wordScript'; export type { FilterConfig }; @@ -25,6 +32,7 @@ export type { FilterConfig }; */ class Filter { private words: Map; + private wordScripts: Map; private caseSensitive: boolean; private wordBoundaries: boolean; private replaceWith?: string; @@ -48,6 +56,7 @@ class Filter { private maxCacheSize: number; private cache: Map; private regexCache: Map; + private dictionaryMatcher: DictionaryAhoCorasick | null; /** * Creates a new Filter instance with the specified configuration. @@ -137,7 +146,37 @@ class Filter { words = [...words, ...config.customWords]; } - this.words = new Map(words.map((word) => [word.toLowerCase(), 1])); + this.words = new Map(); + this.wordScripts = new Map(); + for (const word of words) { + const key = word.toLowerCase(); + this.words.set(key, 1); + this.wordScripts.set(key, classifyWordScript(word)); + } + + this.dictionaryMatcher = this.shouldUseAhoCorasick(config) + ? new DictionaryAhoCorasick(Array.from(this.words.keys())) + : null; + } + + private shouldUseAhoCorasick(config?: FilterConfig): boolean { + if (config?.disableAhoCorasick) { + return false; + } + return this.wordBoundaries || this.enableContextAware; + } + + private getDictionarySearchOptions() { + return { + wordBoundaries: this.wordBoundaries, + caseSensitive: this.caseSensitive, + ignoreWords: this.ignoreWords, + wordScripts: this.wordScripts, + }; + } + + private getWordScript(word: string): WordScript { + return this.wordScripts.get(word) ?? classifyWordScript(word); } private debugLog(...args: unknown[]) { @@ -155,30 +194,109 @@ class Filter { * @returns The normalized text */ private normalizeText(text: string, aggressive: boolean = false): string { - let normalized = text; + const variants = this.getNormalizedVariants(text); + return aggressive ? variants.aggressive : variants.normal; + } + + /** + * Computes normal and aggressive normalized text in one pass (Unicode + leetspeak). + */ + private getNormalizedVariants(text: string): { normal: string; aggressive: string } { + let base = normalizeEvasion(text); - // Step 1: Apply Unicode normalization (handles homoglyphs, diacritics, etc.) if (this.normalizeUnicodeEnabled) { - normalized = normalizeUnicode(normalized); + base = normalizeUnicode(base); } - // Step 2: Apply leetspeak normalization if (this.detectLeetspeak) { - normalized = normalizeLeetspeak(normalized, { + return normalizeLeetspeakVariants(base, { level: this.leetspeakLevel, collapseRepeated: true, - // Keep double letters like "ss" for normal check, collapse all for aggressive - maxRepeated: aggressive ? 1 : 2, removeSpacedChars: true, }); } - // Step 3: Apply legacy obfuscation handling (for backward compatibility) - if (this.allowObfuscatedMatch && !this.detectLeetspeak) { - normalized = this.normalizeObfuscated(normalized); + if (this.allowObfuscatedMatch) { + const obfuscated = this.normalizeObfuscated(base); + return { normal: obfuscated, aggressive: obfuscated }; } - return normalized; + return { normal: base, aggressive: base }; + } + + /** + * Builds the three text variants used for matching (original + normalized + aggressive). + */ + private getTextVariants( + text: string, + lowercase: boolean, + ): { original: string; normalized: string; aggressive: string } { + const { normal, aggressive } = this.getNormalizedVariants(text); + + if (lowercase) { + return { + original: text.toLowerCase(), + normalized: normal.toLowerCase(), + aggressive: aggressive.toLowerCase(), + }; + } + + return { + original: text, + normalized: normal, + aggressive: aggressive, + }; + } + + private evaluateSeverityOnVariants( + word: string, + variants: { original: string; normalized: string; aggressive: string }, + ): SeverityLevel | undefined { + let severity = this.evaluateSeverity(word, variants.original); + if (severity !== undefined) { + return severity; + } + + if (variants.normalized !== variants.original) { + severity = this.evaluateSeverity(word, variants.normalized); + if (severity !== undefined) { + return severity; + } + } + + if (variants.aggressive !== variants.normalized && variants.aggressive !== variants.original) { + severity = this.evaluateSeverity(word, variants.aggressive); + if (severity !== undefined) { + return severity; + } + } + + return undefined; + } + + private collectMatchesFromVariant( + dictWord: string, + variantText: string, + severity: SeverityLevel, + profaneWords: Set, + severityMap: Record, + useMatchText: boolean, + ): void { + const regex = this.getRegex(dictWord); + const script = this.getWordScript(dictWord); + let match: RegExpExecArray | null; + while ((match = regex.exec(variantText)) !== null) { + const start = match.index; + const end = match.index + match[0].length; + if (!matchHasWordBoundary(variantText, start, end, script, this.wordBoundaries)) { + continue; + } + const matched = useMatchText ? match[0] : dictWord; + profaneWords.add(matched); + if (severityMap[matched] === undefined) { + severityMap[matched] = severity; + } + } } /** @@ -295,16 +413,19 @@ class Filter { } private getRegex(word: string): RegExp { - if (this.regexCache.has(word)) { - const regex = this.regexCache.get(word)!; + const script = this.getWordScript(word); + const cacheKey = `${script}:${word}`; + if (this.regexCache.has(cacheKey)) { + const regex = this.regexCache.get(cacheKey)!; regex.lastIndex = 0; return regex; } const flags = this.caseSensitive ? 'g' : 'gi'; const escapedWord = word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const boundary = this.wordBoundaries ? '\\b' : ''; + const useLatinBoundary = this.wordBoundaries && script === 'latin'; + const boundary = useLatinBoundary ? '\\b' : ''; const regex = new RegExp(`${boundary}${escapedWord}${boundary}`, flags); - this.regexCache.set(word, regex); + this.regexCache.set(cacheKey, regex); return regex; } @@ -330,12 +451,24 @@ class Filter { word: string, text: string, ): SeverityLevel | undefined { - // Check for exact word match (with or without word boundaries) - if (this.getRegex(word).test(text)) { + const script = this.getWordScript(word); + const regex = this.getRegex(word); + + if (script === 'cjk' && this.wordBoundaries) { + let match: RegExpExecArray | null; + while ((match = regex.exec(text)) !== null) { + const start = match.index; + const end = match.index + match[0].length; + if (matchHasWordBoundary(text, start, end, script, true)) { + return SeverityLevel.EXACT; + } + } + return undefined; + } + + if (regex.test(text)) { return SeverityLevel.EXACT; } - // Only use fuzzy matching when word boundaries are disabled - // This prevents the Scunthorpe problem (matching "cunt" in "scunthorpe") if (!this.wordBoundaries && this.isFuzzyToleranceMatch(word, text)) { return SeverityLevel.FUZZY; } @@ -359,25 +492,176 @@ class Filter { * ``` */ isProfane(value: string): boolean { - // Check against original, normalized, and aggressively normalized text - const originalInput = value; - const normalizedInput = this.normalizeText(value); - const aggressiveInput = this.normalizeText(value, true); + if (this.enableContextAware) { + return this.isProfaneWithContextAware(value); + } + if (this.dictionaryMatcher) { + return this.isProfaneWithAhoCorasick(value); + } + return this.isProfaneLegacy(value); + } - for (const word of this.words.keys()) { - if (this.ignoreWords.has(word.toLowerCase())) { + private passesContextFilter( + text: string, + matchedWord: string, + matchIndex: number, + ): boolean { + if (!this.contextAnalyzer) { + return true; + } + const contextResult = this.contextAnalyzer.analyzeContext( + text, + matchedWord, + matchIndex, + ); + return !( + contextResult.isWhitelisted || + contextResult.contextScore > this.confidenceThreshold + ); + } + + private isProfaneWithContextAware(value: string): boolean { + const variants = this.getTextVariants(value, false); + + if (this.dictionaryMatcher) { + return this.hasContextAwareAcMatch(value, variants); + } + + return this.hasContextAwareLegacyMatch(value, variants); + } + + private hasContextAwareAcMatch( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + ): boolean { + const options = this.getDictionarySearchOptions(); + const matcher = this.dictionaryMatcher!; + + const hasFlaggedMatch = ( + variantText: string, + useMatchedText: boolean, + ): boolean => { + for (const match of matcher.findMatches(variantText, options)) { + const matchedWord = useMatchedText ? match.matchedText : match.dictWord; + if (this.passesContextFilter(text, matchedWord, match.start)) { + return true; + } + } + return false; + }; + + if (hasFlaggedMatch(variants.original, true)) { + return true; + } + if ( + variants.normalized !== variants.original && + hasFlaggedMatch(variants.normalized, false) + ) { + return true; + } + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original && + hasFlaggedMatch(variants.aggressive, false) + ) { + return true; + } + return false; + } + + private hasContextAwareLegacyMatch( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + ): boolean { + for (const dictWord of this.words.keys()) { + if (this.ignoreWords.has(dictWord.toLowerCase())) { continue; } - // Check against original text first (for raw leetspeak matches like f4ck) - if (this.evaluateSeverity(word, originalInput) !== undefined) { + + if ( + this.hasContextAwareLegacyMatchForWord(text, variants, dictWord) + ) { return true; } - // Check against normalized text (for @ss → ass) - if (this.evaluateSeverity(word, normalizedInput) !== undefined) { - return true; + } + return false; + } + + private hasContextAwareLegacyMatchForWord( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + dictWord: string, + ): boolean { + const checkVariant = (variantText: string, useMatchText: boolean): boolean => { + const severity = this.evaluateSeverity(dictWord, variantText); + if (severity === undefined) { + return false; + } + + const regex = this.getRegex(dictWord); + const script = this.getWordScript(dictWord); + let match: RegExpExecArray | null; + while ((match = regex.exec(variantText)) !== null) { + const start = match.index; + const end = match.index + match[0].length; + if (!matchHasWordBoundary(variantText, start, end, script, this.wordBoundaries)) { + continue; + } + const matchedWord = useMatchText ? match[0] : dictWord; + if (this.passesContextFilter(text, matchedWord, start)) { + return true; + } } - // Check against aggressive normalization (for fuuuuck → fuck) - if (this.evaluateSeverity(word, aggressiveInput) !== undefined) { + return false; + }; + + if (checkVariant(variants.original, true)) { + return true; + } + if (variants.normalized !== variants.original && checkVariant(variants.normalized, false)) { + return true; + } + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original && + checkVariant(variants.aggressive, false) + ) { + return true; + } + return false; + } + + private isProfaneWithAhoCorasick(value: string): boolean { + const variants = this.getTextVariants(value, false); + const options = this.getDictionarySearchOptions(); + + if (this.dictionaryMatcher!.hasAnyMatch(variants.original, options)) { + return true; + } + if ( + variants.normalized !== variants.original && + this.dictionaryMatcher!.hasAnyMatch(variants.normalized, options) + ) { + return true; + } + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original && + this.dictionaryMatcher!.hasAnyMatch(variants.aggressive, options) + ) { + return true; + } + return false; + } + + private isProfaneLegacy(value: string): boolean { + const variants = this.getTextVariants(value, false); + + for (const word of this.words.keys()) { + if (this.ignoreWords.has(word.toLowerCase())) { + continue; + } + if (this.evaluateSeverityOnVariants(word, variants) !== undefined) { return true; } } @@ -388,202 +672,401 @@ class Filter { return this.isProfane(word); } - /** - * Performs a comprehensive profanity check on the given text. - * - * @param text - The text to check for profanity - * @returns Result object containing detected profanity information - * - * @example - * ```typescript - * const filter = new Filter({ - * languages: ['english'], - * detectLeetspeak: true, - * normalizeUnicode: true, - * }); - * - * const result = filter.checkProfanity('This is f4ck!ng bad'); - * console.log(result.containsProfanity); // true - * console.log(result.profaneWords); // ['fuck'] - * - * // With caching for repeated checks - * const filter2 = new Filter({ cacheResults: true }); - * filter2.checkProfanity('same text'); // Computed - * filter2.checkProfanity('same text'); // Retrieved from cache - * ``` - */ - checkProfanity(text: string): CheckProfanityResult { - // Check cache first - const cacheKey = text; - const cachedResult = this.getFromCache(cacheKey); - if (cachedResult) { - this.debugLog('Cache hit for:', text.substring(0, 50)); - return cachedResult; + private checkProfanityWithAhoCorasick(text: string): CheckProfanityResult { + const variants = this.getTextVariants(text, true); + const profaneWords = new Set(); + const severityMap: Record = {}; + const options = this.getDictionarySearchOptions(); + + for (const match of this.dictionaryMatcher!.findMatches(variants.original, options)) { + profaneWords.add(match.matchedText); + if (severityMap[match.matchedText] === undefined) { + severityMap[match.matchedText] = SeverityLevel.EXACT; + } } - // Backward compatibility: if not context-aware, run old logic - if (!this.enableContextAware) { - // Check original, normalized, and aggressively normalized text - const originalInput = text.toLowerCase(); - const normalizedInput = this.normalizeText(text).toLowerCase(); - const aggressiveInput = this.normalizeText(text, true).toLowerCase(); + if (variants.normalized !== variants.original) { + for (const match of this.dictionaryMatcher!.findMatches(variants.normalized, options)) { + profaneWords.add(match.dictWord); + if (severityMap[match.dictWord] === undefined) { + severityMap[match.dictWord] = SeverityLevel.EXACT; + } + } + } - const profaneWords: string[] = []; - const severityMap: Record = {}; + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original + ) { + for (const match of this.dictionaryMatcher!.findMatches(variants.aggressive, options)) { + profaneWords.add(match.dictWord); + if (severityMap[match.dictWord] === undefined) { + severityMap[match.dictWord] = SeverityLevel.EXACT; + } + } + } - for (const dictWord of this.words.keys()) { - if (this.ignoreWords.has(dictWord.toLowerCase())) continue; + return this.buildProfanityResult(text, profaneWords, severityMap); + } - // Check against original text first (for raw leetspeak matches like f4ck) - let severity = this.evaluateSeverity(dictWord, originalInput); - if (severity !== undefined) { - const regex = this.getRegex(dictWord); - let match; - while ((match = regex.exec(originalInput)) !== null) { - profaneWords.push(match[0]); - if (severityMap[match[0]] === undefined) { - severityMap[match[0]] = severity; - } - } - } + private checkProfanityLegacyNonContext(text: string): CheckProfanityResult { + const variants = this.getTextVariants(text, true); + const profaneWords = new Set(); + const severityMap: Record = {}; + + for (const dictWord of this.words.keys()) { + if (this.ignoreWords.has(dictWord.toLowerCase())) continue; + + let severity = this.evaluateSeverity(dictWord, variants.original); + if (severity !== undefined) { + this.collectMatchesFromVariant( + dictWord, + variants.original, + severity, + profaneWords, + severityMap, + true, + ); + } - // Check against normalized text (for @ss → ass) - severity = this.evaluateSeverity(dictWord, normalizedInput); + if (variants.normalized !== variants.original) { + severity = this.evaluateSeverity(dictWord, variants.normalized); if (severity !== undefined) { - const regex = this.getRegex(dictWord); - let match; - while ((match = regex.exec(normalizedInput)) !== null) { - if (!profaneWords.includes(dictWord)) { - profaneWords.push(dictWord); - if (severityMap[dictWord] === undefined) { - severityMap[dictWord] = severity; - } - } - } + this.collectMatchesFromVariant( + dictWord, + variants.normalized, + severity, + profaneWords, + severityMap, + false, + ); } + } - // Check against aggressive normalization (for fuuuuck → fuck) - severity = this.evaluateSeverity(dictWord, aggressiveInput); + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original + ) { + severity = this.evaluateSeverity(dictWord, variants.aggressive); if (severity !== undefined) { - if (!profaneWords.includes(dictWord)) { - profaneWords.push(dictWord); - if (severityMap[dictWord] === undefined) { - severityMap[dictWord] = severity; - } + profaneWords.add(dictWord); + if (severityMap[dictWord] === undefined) { + severityMap[dictWord] = SeverityLevel.EXACT; } } } + } - let processedText = text; - if (this.replaceWith && profaneWords.length > 0) { - const uniqueWords = Array.from(new Set(profaneWords)); - for (const word of uniqueWords) { - const escaped = word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const replacementRegex = this.wordBoundaries - ? new RegExp(`\\b${escaped}\\b`, 'gi') - : new RegExp(escaped, 'gi'); - processedText = processedText.replace( - replacementRegex, - this.replaceWith, - ); - } + return this.buildProfanityResult(text, profaneWords, severityMap); + } + + private recordContextAwareMatch( + text: string, + matchedWord: string, + matchIndex: number, + severity: SeverityLevel, + profaneWords: string[], + severityMap: Record, + matches: Match[], + seen: Set, + ): void { + const dedupeKey = `${matchedWord}:${matchIndex}`; + if (seen.has(dedupeKey)) { + return; + } + + const matchObj: Match = { + word: matchedWord, + index: matchIndex, + severity, + }; + + if (this.contextAnalyzer) { + const contextResult = this.contextAnalyzer.analyzeContext( + text, + matchedWord, + matchIndex, + ); + matchObj.contextScore = contextResult.contextScore; + matchObj.reason = contextResult.reason; + matchObj.isWhitelisted = contextResult.isWhitelisted; + if (!this.passesContextFilter(text, matchedWord, matchIndex)) { + return; } + } - const result: CheckProfanityResult = { - containsProfanity: profaneWords.length > 0, - profaneWords: Array.from(new Set(profaneWords)), - processedText: this.replaceWith ? processedText : undefined, - severityMap: - this.severityLevels && Object.keys(severityMap).length > 0 - ? severityMap - : undefined, - }; + seen.add(dedupeKey); + profaneWords.push(matchedWord); + if (severityMap[matchedWord] === undefined) { + severityMap[matchedWord] = severity; + } + matches.push(matchObj); + } - // Cache the result - this.addToCache(cacheKey, result); - return result; + private collectContextAwareCandidatesFromAc( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + profaneWords: string[], + severityMap: Record, + matches: Match[], + seen: Set, + ): void { + const options = this.getDictionarySearchOptions(); + const matcher = this.dictionaryMatcher!; + + const processAcMatch = (match: DictionaryMatch, useMatchedText: boolean) => { + this.recordContextAwareMatch( + text, + useMatchedText ? match.matchedText : match.dictWord, + match.start, + SeverityLevel.EXACT, + profaneWords, + severityMap, + matches, + seen, + ); + }; + + for (const match of matcher.findMatches(variants.original, options)) { + processAcMatch(match, true); } - // Context-aware path - // Apply all normalizations - let input = this.normalizeText(text); - input = input.toLowerCase(); - const originalText = text; - const profaneWords: string[] = []; - const severityMap: Record = {}; - const matches: Match[] = []; + if (variants.normalized !== variants.original) { + for (const match of matcher.findMatches(variants.normalized, options)) { + processAcMatch(match, false); + } + } + + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original + ) { + for (const match of matcher.findMatches(variants.aggressive, options)) { + processAcMatch(match, false); + } + } + } + private collectContextAwareCandidatesFromLegacy( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + profaneWords: string[], + severityMap: Record, + matches: Match[], + seen: Set, + ): void { for (const dictWord of this.words.keys()) { - if (this.ignoreWords.has(dictWord.toLowerCase())) continue; - const severity = this.evaluateSeverity(dictWord, input); - if (severity !== undefined) { + if (this.ignoreWords.has(dictWord.toLowerCase())) { + continue; + } + + const collectFromVariant = ( + variantText: string, + useMatchText: boolean, + ) => { + const severity = this.evaluateSeverity(dictWord, variantText); + if (severity === undefined) { + return; + } + const regex = this.getRegex(dictWord); - let match; - while ((match = regex.exec(input)) !== null) { - const matchedWord = match[0]; - const matchIndex = match.index; - const matchObj: Match = { - word: matchedWord, - index: matchIndex, - severity: severity - }; - if (this.enableContextAware && this.contextAnalyzer) { - const contextResult = this.contextAnalyzer.analyzeContext( - originalText, - matchedWord, - matchIndex - ); - matchObj.contextScore = contextResult.contextScore; - matchObj.reason = contextResult.reason; - matchObj.isWhitelisted = contextResult.isWhitelisted; - if (contextResult.isWhitelisted || (contextResult.contextScore > this.confidenceThreshold)) { - continue; - } - } - profaneWords.push(matchedWord); - if (severityMap[matchedWord] === undefined) { - severityMap[matchedWord] = severity; + const script = this.getWordScript(dictWord); + let match: RegExpExecArray | null; + while ((match = regex.exec(variantText)) !== null) { + const start = match.index; + const end = match.index + match[0].length; + if (!matchHasWordBoundary(variantText, start, end, script, this.wordBoundaries)) { + continue; } - matches.push(matchObj); + this.recordContextAwareMatch( + text, + useMatchText ? match[0] : dictWord, + start, + severity, + profaneWords, + severityMap, + matches, + seen, + ); } + }; + + collectFromVariant(variants.original, true); + + if (variants.normalized !== variants.original) { + collectFromVariant(variants.normalized, false); + } + + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original + ) { + collectFromVariant(variants.aggressive, false); } } - if (profaneWords.length > 0) { - this.debugLog('Detected:', profaneWords); - } + } + + private buildContextAwareResult( + text: string, + profaneWords: string[], + severityMap: Record, + matches: Match[], + ): CheckProfanityResult { let processedText = text; + if (this.replaceWith && profaneWords.length > 0) { const uniqueWords = Array.from(new Set(profaneWords)); for (const word of uniqueWords) { - const escaped = word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const replacementRegex = this.wordBoundaries - ? new RegExp(`\\b${escaped}\\b`, 'gi') - : new RegExp(escaped, 'gi'); processedText = processedText.replace( - replacementRegex, + this.getReplacementRegex(word), this.replaceWith, ); } } + let contextScore: number | undefined; - if (this.enableContextAware && matches.length > 0) { - const totalScore = matches.reduce((sum, match) => - sum + (match.contextScore || 0.5), 0); + if (matches.length > 0) { + const totalScore = matches.reduce( + (sum, match) => sum + (match.contextScore || 0.5), + 0, + ); contextScore = totalScore / matches.length; } - const result: CheckProfanityResult = { + + return { containsProfanity: profaneWords.length > 0, profaneWords: Array.from(new Set(profaneWords)), processedText: this.replaceWith ? processedText : undefined, - severityMap: this.severityLevels && Object.keys(severityMap).length > 0 ? severityMap : undefined, + severityMap: + this.severityLevels && Object.keys(severityMap).length > 0 + ? severityMap + : undefined, matches: matches.length > 0 ? matches : undefined, contextScore, - reason: matches.length > 0 ? - `Found ${matches.length} potential profanity matches` : - 'No profanity detected' + reason: + matches.length > 0 + ? `Found ${matches.length} potential profanity matches` + : 'No profanity detected', }; + } + + private checkProfanityWithContextAware(text: string): CheckProfanityResult { + const variants = this.getTextVariants(text, true); + const profaneWords: string[] = []; + const severityMap: Record = {}; + const matches: Match[] = []; + const seen = new Set(); + + if (this.dictionaryMatcher) { + this.collectContextAwareCandidatesFromAc( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + ); + } else { + this.collectContextAwareCandidatesFromLegacy( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + ); + } + + if (profaneWords.length > 0) { + this.debugLog('Detected:', profaneWords); + } + + return this.buildContextAwareResult(text, profaneWords, severityMap, matches); + } + + private getReplacementRegex(word: string): RegExp { + const escaped = word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + if (!this.wordBoundaries) { + return new RegExp(escaped, 'gi'); + } + const script = classifyWordScript(word); + if (script === 'cjk') { + return new RegExp(escaped, 'gi'); + } + return new RegExp(`\\b${escaped}\\b`, 'gi'); + } + + private buildProfanityResult( + text: string, + profaneWords: Set, + severityMap: Record, + ): CheckProfanityResult { + const profaneWordList = Array.from(profaneWords); + let processedText = text; + + if (this.replaceWith && profaneWordList.length > 0) { + for (const word of profaneWordList) { + processedText = processedText.replace( + this.getReplacementRegex(word), + this.replaceWith, + ); + } + } + + return { + containsProfanity: profaneWordList.length > 0, + profaneWords: profaneWordList, + processedText: this.replaceWith ? processedText : undefined, + severityMap: + this.severityLevels && Object.keys(severityMap).length > 0 + ? severityMap + : undefined, + }; + } + + /** + * Performs a comprehensive profanity check on the given text. + * + * @param text - The text to check for profanity + * @returns Result object containing detected profanity information + * + * @example + * ```typescript + * const filter = new Filter({ + * languages: ['english'], + * detectLeetspeak: true, + * normalizeUnicode: true, + * }); + * + * const result = filter.checkProfanity('This is f4ck!ng bad'); + * console.log(result.containsProfanity); // true + * console.log(result.profaneWords); // ['fuck'] + * + * // With caching for repeated checks + * const filter2 = new Filter({ cacheResults: true }); + * filter2.checkProfanity('same text'); // Computed + * filter2.checkProfanity('same text'); // Retrieved from cache + * ``` + */ + checkProfanity(text: string): CheckProfanityResult { + // Check cache first + const cacheKey = text; + const cachedResult = this.getFromCache(cacheKey); + if (cachedResult) { + this.debugLog('Cache hit for:', text.substring(0, 50)); + return cachedResult; + } + + // Backward compatibility: if not context-aware, run old logic + if (!this.enableContextAware) { + const result = this.dictionaryMatcher + ? this.checkProfanityWithAhoCorasick(text) + : this.checkProfanityLegacyNonContext(text); + this.addToCache(cacheKey, result); + return result; + } - // Cache the result + const result = this.checkProfanityWithContextAware(text); this.addToCache(cacheKey, result); return result; } diff --git a/packages/js/src/filters/dictionaryAhoCorasick.ts b/packages/js/src/filters/dictionaryAhoCorasick.ts new file mode 100644 index 0000000..7292e4c --- /dev/null +++ b/packages/js/src/filters/dictionaryAhoCorasick.ts @@ -0,0 +1,128 @@ +import AhoCorasick from 'modern-ahocorasick'; +import { + matchHasWordBoundary, + type WordScript, +} from '../utils/wordScript'; + +/** Grapheme segmenter — runtime API exists in Node 18+ / modern browsers. */ +type GraphemeSegment = { segment: string; index: number }; +type GraphemeSegmenterInstance = { segment(input: string): Iterable }; +type GraphemeSegmenterConstructor = new ( + locales?: string | string[], + options?: { granularity: 'grapheme' }, +) => GraphemeSegmenterInstance; + +const GraphemeSegmenter = ( + Intl as unknown as { Segmenter: GraphemeSegmenterConstructor } +).Segmenter; + +export interface DictionaryMatch { + dictWord: string; + start: number; + end: number; + matchedText: string; +} + +export interface DictionarySearchOptions { + wordBoundaries: boolean; + caseSensitive: boolean; + ignoreWords: Set; + wordScripts: Map; +} + +const graphemeSegmenter = new GraphemeSegmenter(undefined, { granularity: 'grapheme' }); + +function countGraphemes(text: string): number { + let count = 0; + for (const _ of graphemeSegmenter.segment(text)) { + count++; + } + return count; +} + +function graphemeStartToStringIndex(text: string, graphemeIndex: number): number { + let i = 0; + for (const seg of graphemeSegmenter.segment(text)) { + if (i === graphemeIndex) { + return seg.index; + } + i++; + } + return text.length; +} + +function graphemeEndToExclusiveStringIndex( + text: string, + endGraphemeIndexInclusive: number, +): number { + let i = 0; + for (const seg of graphemeSegmenter.segment(text)) { + if (i === endGraphemeIndexInclusive) { + return seg.index + seg.segment.length; + } + i++; + } + return text.length; +} + +/** + * Multi-pattern dictionary matcher backed by the Aho-Corasick algorithm. + * Used for exact matching when word boundaries are enabled (no fuzzy path). + */ +export class DictionaryAhoCorasick { + private readonly ac: AhoCorasick; + private readonly wordGraphemeLengths: Map; + + constructor(words: string[]) { + this.ac = new AhoCorasick(words); + this.wordGraphemeLengths = new Map( + words.map((word) => [word, countGraphemes(word)]), + ); + } + + hasAnyMatch(text: string, options: DictionarySearchOptions): boolean { + return this.findMatches(text, options).length > 0; + } + + findMatches(text: string, options: DictionarySearchOptions): DictionaryMatch[] { + const haystack = options.caseSensitive ? text : text.toLowerCase(); + const hits = this.ac.search(haystack); + const results: DictionaryMatch[] = []; + const seen = new Set(); + + for (const [endGraphemeIdx, dictWords] of hits) { + for (const dictWord of dictWords) { + if (options.ignoreWords.has(dictWord.toLowerCase())) { + continue; + } + + const wordLen = this.wordGraphemeLengths.get(dictWord) ?? dictWord.length; + const startGrapheme = endGraphemeIdx - wordLen + 1; + const start = graphemeStartToStringIndex(text, startGrapheme); + const end = graphemeEndToExclusiveStringIndex(text, endGraphemeIdx); + const script = options.wordScripts.get(dictWord) ?? 'latin'; + + if ( + !matchHasWordBoundary(text, start, end, script, options.wordBoundaries) + ) { + continue; + } + + const dedupeKey = `${dictWord}:${start}:${end}`; + if (seen.has(dedupeKey)) { + continue; + } + seen.add(dedupeKey); + + results.push({ + dictWord, + start, + end, + matchedText: text.slice(start, end), + }); + } + } + + return results; + } +} diff --git a/packages/js/src/hooks/useProfanityChecker.ts b/packages/js/src/hooks/useProfanityChecker.ts index 7139595..82aa844 100644 --- a/packages/js/src/hooks/useProfanityChecker.ts +++ b/packages/js/src/hooks/useProfanityChecker.ts @@ -1,5 +1,6 @@ -import { useState, useCallback } from 'react'; -import { checkProfanity, checkProfanityAsync, isWordProfane } from '../core'; +import { useState, useCallback, useRef, useEffect } from 'react'; +import { checkProfanity, checkProfanityAsync } from '../core'; +import { createFilterConfig, getPooledFilter } from '../core/filterPool'; import type { ProfanityCheckerConfig } from '../core/types'; import type { CheckProfanityResult } from '../types/types'; @@ -7,6 +8,11 @@ export type { ProfanityCheckerConfig }; export const useProfanityChecker = (config?: ProfanityCheckerConfig) => { const [result, setResult] = useState(null); + const filterRef = useRef(getPooledFilter(createFilterConfig(config))); + + useEffect(() => { + filterRef.current = getPooledFilter(createFilterConfig(config)); + }, [config]); const checkText = useCallback((text: string) => { const checkResult = checkProfanity(text, config); @@ -21,8 +27,8 @@ export const useProfanityChecker = (config?: ProfanityCheckerConfig) => { }, [config]); const isWordProfaneCallback = useCallback((word: string) => { - return isWordProfane(word, config); - }, [config]); + return filterRef.current.isProfane(word); + }, []); const reset = useCallback(() => setResult(null), []); diff --git a/packages/js/src/scanners/secrets.ts b/packages/js/src/scanners/secrets.ts index bfb10af..8c83a1b 100644 --- a/packages/js/src/scanners/secrets.ts +++ b/packages/js/src/scanners/secrets.ts @@ -71,6 +71,22 @@ export interface SecretsOptions { // SecretsScanner // --------------------------------------------------------------------------- +/** A secret pattern with a pre-compiled global RegExp for scanning. */ +interface CompiledSecretPattern { + def: SecretPattern; + re: RegExp; +} + +function compileSecretPattern(entry: SecretPattern): CompiledSecretPattern { + const flags = entry.pattern.flags.includes('g') + ? entry.pattern.flags + : entry.pattern.flags + 'g'; + return { + def: entry, + re: new RegExp(entry.pattern.source, flags), + }; +} + /** * Scanner that detects secrets, API keys, and credentials in text. * @@ -80,13 +96,14 @@ export class SecretsScanner implements Scanner { /** @inheritdoc */ readonly name = 'secrets'; - private readonly patterns: SecretPattern[]; + private readonly compiledPatterns: CompiledSecretPattern[]; private readonly options: Required> & { vault?: Vault; }; constructor(options: SecretsOptions = {}) { - this.patterns = [...SECRET_PATTERNS, ...(options.customPatterns ?? [])]; + const patterns = [...SECRET_PATTERNS, ...(options.customPatterns ?? [])]; + this.compiledPatterns = patterns.map(compileSecretPattern); this.options = { redact: options.redact ?? false, vault: options.vault, @@ -103,12 +120,9 @@ export class SecretsScanner implements Scanner { const reasons: string[] = []; let sanitized = input; - for (const entry of this.patterns) { - const flags = entry.pattern.flags.includes('g') - ? entry.pattern.flags - : entry.pattern.flags + 'g'; - const re = new RegExp(entry.pattern.source, flags); + for (const { def: entry, re } of this.compiledPatterns) { let m: RegExpExecArray | null; + re.lastIndex = 0; while ((m = re.exec(input)) !== null) { // For patterns with capture groups, prefer group 1 for entropy check diff --git a/packages/js/src/types/types.ts b/packages/js/src/types/types.ts index cba5d11..d1844d0 100644 --- a/packages/js/src/types/types.ts +++ b/packages/js/src/types/types.ts @@ -114,6 +114,13 @@ export interface FilterConfig extends ContextAwareConfig { * @default 1000 */ maxCacheSize?: number; + + /** + * Disable the Aho-Corasick fast path and use regex scanning only. + * Intended for tests and parity validation — not for production use. + * @internal + */ + disableAhoCorasick?: boolean; } /** Result with minimum severity filtering */ diff --git a/packages/js/src/utils/evasion.ts b/packages/js/src/utils/evasion.ts new file mode 100644 index 0000000..b2f71e3 --- /dev/null +++ b/packages/js/src/utils/evasion.ts @@ -0,0 +1,63 @@ +/** + * @fileoverview Evasion normalization for profanity detection. + * Handles HTML injection, separator obfuscation, and asterisk masking. + * @module utils/evasion + */ + +/** + * Removes HTML tags and decodes common numeric/named entities. + */ +export function stripHtmlAndDecodeEntities(text: string): string { + let result = text.replace(/<[^>]*>/g, ''); + + result = result.replace(/&#(\d+);/g, (_, dec: string) => { + const code = parseInt(dec, 10); + return Number.isFinite(code) ? String.fromCharCode(code) : _; + }); + + result = result.replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) => { + const code = parseInt(hex, 16); + return Number.isFinite(code) ? String.fromCharCode(code) : _; + }); + + return result + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/&/gi, '&') + .replace(/"/gi, '"') + .replace(/'/gi, "'"); +} + +/** + * Collapses single alphanumerics separated by spaces, dots, underscores, or hyphens. + * Handles patterns like "f.u.c.k", "f_u_c_k", and "b-i-t-c-h". + */ +export function collapseSeparatedCharacters(text: string): string { + const pattern = + /\b([a-zA-Z0-9@$!#*])(?:[\s._\-]+([a-zA-Z0-9@$!#*])){2,}\b/g; + + return text.replace(pattern, (match) => match.replace(/[\s._\-]+/g, '')); +} + +/** + * Expands common asterisk-masked profanity abbreviations. + */ +export function normalizeMaskedProfanity(text: string): string { + return text + .replace(/\bf\*+cking\b/gi, 'fucking') + .replace(/\bf\*+ck\b/gi, 'fuck') + .replace(/\bs\*+hit\b/gi, 'shit') + .replace(/\bf\*{2,}(?=\W|$)/gi, 'fuck') + .replace(/\bf\s+yourself\b/gi, 'fuck yourself'); +} + +/** + * Applies all evasion normalization steps before Unicode/leetspeak handling. + */ +export function normalizeEvasion(text: string): string { + let result = text; + result = stripHtmlAndDecodeEntities(result); + result = collapseSeparatedCharacters(result); + result = normalizeMaskedProfanity(result); + return result; +} diff --git a/packages/js/src/utils/index.ts b/packages/js/src/utils/index.ts index ce409e8..70dca93 100644 --- a/packages/js/src/utils/index.ts +++ b/packages/js/src/utils/index.ts @@ -3,6 +3,13 @@ * @module utils */ +export { + normalizeEvasion, + stripHtmlAndDecodeEntities, + collapseSeparatedCharacters, + normalizeMaskedProfanity, +} from './evasion'; + export { normalizeLeetspeak, collapseSpacedCharacters, @@ -23,3 +30,12 @@ export { detectCharacterSets, type UnicodeNormalizationOptions, } from './unicode'; + +export { + classifyWordScript, + hasCjkWordBoundary, + hasLatinWordBoundary, + isCjkCharacter, + matchHasWordBoundary, + type WordScript, +} from './wordScript'; diff --git a/packages/js/src/utils/leetspeak.ts b/packages/js/src/utils/leetspeak.ts index 49ee771..aa2e1b6 100644 --- a/packages/js/src/utils/leetspeak.ts +++ b/packages/js/src/utils/leetspeak.ts @@ -171,39 +171,105 @@ const AGGRESSIVE_SUBSTITUTIONS: Record = { * normalizeLeetspeak('fuuuuck'); // Returns: 'fuck' * ``` */ -export function normalizeLeetspeak( +const AGGRESSIVE_VOWEL_SUBSTITUTIONS: Record = { + ...AGGRESSIVE_SUBSTITUTIONS, + '@': 'u', +}; + +function applyCharSubstitutionsWithMap( text: string, - options: LeetspeakOptions = {} + substitutions: Record, ): string { - const { - level = 'moderate', - collapseRepeated = true, - maxRepeated = 2, - removeSpacedChars = true, - } = options; + const chars: string[] = []; + for (let i = 0; i < text.length; i++) { + const char = text[i]; + chars.push(substitutions[char] ?? char); + } + return chars.join(''); +} +function applyLeetspeakPreCollapseWithMap( + text: string, + options: Pick, + substitutions: Record, +): string { + const { level = 'moderate', removeSpacedChars = true } = options; let normalized = text; - // Step 1: Handle spaced characters (f u c k → fuck) if (removeSpacedChars) { normalized = collapseSpacedCharacters(normalized); } - // Step 2: Apply multi-character patterns first (aggressive only) if (level === 'aggressive') { for (const [pattern, replacement] of AGGRESSIVE_MULTI_CHAR) { normalized = normalized.replace(pattern, replacement); } } - // Step 3: Apply single-character substitutions - const substitutions = getSubstitutionMap(level); - normalized = normalized - .split('') - .map((char) => substitutions[char] || char) - .join(''); + return applyCharSubstitutionsWithMap(normalized, substitutions); +} + +/** + * Applies leetspeak steps 1–3 (spacing, multi-char, char substitutions) without + * collapsing repeated characters. Used to derive normal/aggressive variants in one pass. + */ +function applyLeetspeakPreCollapse( + text: string, + options: Pick = {}, +): string { + const { level = 'moderate', removeSpacedChars = true } = options; + let normalized = text; + + if (removeSpacedChars) { + normalized = collapseSpacedCharacters(normalized); + } + + if (level === 'aggressive') { + for (const [pattern, replacement] of AGGRESSIVE_MULTI_CHAR) { + normalized = normalized.replace(pattern, replacement); + } + } + + return applyCharSubstitutionsWithMap(normalized, getSubstitutionMap(level)); +} + +/** + * Normalizes leetspeak with both standard and aggressive repeat collapsing in one pass. + */ +export function normalizeLeetspeakVariants( + text: string, + options: Omit = {}, +): { normal: string; aggressive: string } { + const { level = 'moderate', collapseRepeated = true } = options; + const preCollapsed = applyLeetspeakPreCollapse(text, options); + const vowelPreCollapsed = + level === 'aggressive' + ? applyLeetspeakPreCollapseWithMap(text, options, AGGRESSIVE_VOWEL_SUBSTITUTIONS) + : preCollapsed; + + if (!collapseRepeated) { + return { normal: preCollapsed, aggressive: vowelPreCollapsed }; + } + + return { + normal: collapseRepeatedCharacters(preCollapsed, 2), + aggressive: collapseRepeatedCharacters(vowelPreCollapsed, 1), + }; +} + +export function normalizeLeetspeak( + text: string, + options: LeetspeakOptions = {} +): string { + const { + level = 'moderate', + collapseRepeated = true, + maxRepeated = 2, + removeSpacedChars = true, + } = options; + + let normalized = applyLeetspeakPreCollapse(text, { level, removeSpacedChars }); - // Step 4: Collapse repeated characters (fuuuuck → fuck) if (collapseRepeated) { normalized = collapseRepeatedCharacters(normalized, maxRepeated); } diff --git a/packages/js/src/utils/unicode.ts b/packages/js/src/utils/unicode.ts index 74d2349..f3c76af 100644 --- a/packages/js/src/utils/unicode.ts +++ b/packages/js/src/utils/unicode.ts @@ -94,6 +94,8 @@ const HOMOGLYPHS: Record = { 'τ': 't', // Greek small tau 'Τ': 'T', // Greek capital Tau 'υ': 'u', // Greek small upsilon + 'Ս': 'U', // Armenian capital seh (looks like U) + 'ս': 'u', // Armenian small seh (looks like u) 'Υ': 'Y', // Greek capital Upsilon 'χ': 'x', // Greek small chi 'Χ': 'X', // Greek capital Chi diff --git a/packages/js/src/utils/wordScript.ts b/packages/js/src/utils/wordScript.ts new file mode 100644 index 0000000..2ea9458 --- /dev/null +++ b/packages/js/src/utils/wordScript.ts @@ -0,0 +1,88 @@ +/** + * Word-script classification and boundary checks for Latin vs CJK dictionary entries. + * + * Latin entries use JavaScript `\b` semantics (ASCII word chars only). + * CJK entries use substring matching when word boundaries are enabled (no `\b`). + */ + +export type WordScript = 'latin' | 'cjk'; + +/** Mirrors JavaScript `\b` word character class without the `u` flag. */ +const LATIN_WORD_CHAR = /[A-Za-z0-9_]/; + +/** + * Returns true when the code point belongs to a CJK-related script block. + */ +export function isCjkCharacter(char: string): boolean { + const code = char.codePointAt(0); + if (code === undefined) { + return false; + } + + return ( + (code >= 0x4e00 && code <= 0x9fff) || // CJK Unified Ideographs + (code >= 0x3400 && code <= 0x4dbf) || // Extension A + (code >= 0x3040 && code <= 0x309f) || // Hiragana + (code >= 0x30a0 && code <= 0x30ff) || // Katakana + (code >= 0x31f0 && code <= 0x31ff) || // Katakana phonetic extensions + (code >= 0xac00 && code <= 0xd7af) || // Hangul syllables + (code >= 0x1100 && code <= 0x11ff) || // Hangul Jamo + (code >= 0x3130 && code <= 0x318f) || // Hangul compatibility Jamo + (code >= 0x3100 && code <= 0x312f) || // Bopomofo + (code >= 0xff66 && code <= 0xff9f) // Halfwidth katakana + ); +} + +/** + * Classify a dictionary entry by its characters (not by configured language). + * ASCII-only entries from Japanese/Chinese lists still use Latin `\b` rules. + */ +export function classifyWordScript(word: string): WordScript { + for (const char of word) { + if (isCjkCharacter(char)) { + return 'cjk'; + } + } + return 'latin'; +} + +/** JavaScript `\b` boundary at index (between word and non-word ASCII chars). */ +export function isLatinWordBoundaryBefore(text: string, index: number): boolean { + const leftIsWord = index > 0 && LATIN_WORD_CHAR.test(text[index - 1]!); + const rightIsWord = index < text.length && LATIN_WORD_CHAR.test(text[index]!); + return leftIsWord !== rightIsWord; +} + +export function hasLatinWordBoundary(text: string, start: number, end: number): boolean { + return ( + isLatinWordBoundaryBefore(text, start) && isLatinWordBoundaryBefore(text, end) + ); +} + +/** + * CJK boundary: substring match anywhere (including ASCII-adjacent / wrapped text). + * Latin `\b` does not apply to CJK scripts; adjacency checks would miss obfuscation + * such as `hello他妈的`, `x乳x`, or `123エッチ456`. + */ +export function hasCjkWordBoundary(_text: string, _start: number, _end: number): boolean { + return true; +} + +/** + * Returns whether a match at [start, end) satisfies boundary rules for its script. + */ +export function matchHasWordBoundary( + text: string, + start: number, + end: number, + script: WordScript, + wordBoundariesEnabled: boolean, +): boolean { + if (!wordBoundariesEnabled) { + return true; + } + if (script === 'cjk') { + return hasCjkWordBoundary(text, start, end); + } + return hasLatinWordBoundary(text, start, end); +} diff --git a/packages/js/tests/aho-corasick-parity.test.ts b/packages/js/tests/aho-corasick-parity.test.ts new file mode 100644 index 0000000..aef34c5 --- /dev/null +++ b/packages/js/tests/aho-corasick-parity.test.ts @@ -0,0 +1,187 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { Filter } from '../src/filters/Filter'; +import type { FilterConfig } from '../src/types/types'; + +interface TortureCase { + input: string; + shouldFlag: boolean; + category?: string; +} + +const tortureSet: TortureCase[] = JSON.parse( + readFileSync( + join(__dirname, '../../../benchmarks/shootout/torture-set.json'), + 'utf8', + ), +); + +const FILTER_CONFIGS: FilterConfig[] = [ + { languages: ['english'] }, + { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'moderate', + }, + { + languages: ['english'], + detectLeetspeak: true, + leetspeakLevel: 'aggressive', + normalizeUnicode: true, + }, + { + languages: ['english', 'spanish'], + detectLeetspeak: true, + normalizeUnicode: true, + }, + { + languages: ['english'], + replaceWith: '***', + detectLeetspeak: true, + severityLevels: true, + }, + { allLanguages: true, detectLeetspeak: true, normalizeUnicode: true }, +]; + +const EXTRA_TEXTS = [ + '', + 'hello world', + 'The quick brown fox', + 'Scunthorpe is a town', + 'classic music', + 'assassin', + 'fuck', + 'FUCK', + 'f4ck', + '@ss', + 'fuuuuck', + 'f u c k', + 'This is damn bad', +]; + +function normalizeResult(result: ReturnType) { + return { + containsProfanity: result.containsProfanity, + profaneWords: [...result.profaneWords].sort(), + processedText: result.processedText, + severityMap: result.severityMap, + }; +} + +describe('Aho-Corasick parity with legacy regex path', () => { + for (const config of FILTER_CONFIGS) { + const label = JSON.stringify(config); + + describe(`config ${label}`, () => { + const fastFilter = new Filter(config); + const legacyFilter = new Filter({ ...config, disableAhoCorasick: true }); + + test('uses Aho-Corasick when eligible', () => { + const expectsAc = + (config.wordBoundaries ?? true) || Boolean(config.enableContextAware); + if (expectsAc && !config.disableAhoCorasick) { + expect(fastFilter).toBeDefined(); + } + }); + + for (const text of EXTRA_TEXTS) { + test(`isProfane parity for ${JSON.stringify(text)}`, () => { + expect(fastFilter.isProfane(text)).toBe(legacyFilter.isProfane(text)); + }); + + test(`checkProfanity parity for ${JSON.stringify(text)}`, () => { + expect(normalizeResult(fastFilter.checkProfanity(text))).toEqual( + normalizeResult(legacyFilter.checkProfanity(text)), + ); + }); + } + + for (const tortureCase of tortureSet) { + test(`torture-set [${tortureCase.category}] ${JSON.stringify(tortureCase.input)}`, () => { + expect(fastFilter.isProfane(tortureCase.input)).toBe( + legacyFilter.isProfane(tortureCase.input), + ); + expect(normalizeResult(fastFilter.checkProfanity(tortureCase.input))).toEqual( + normalizeResult(legacyFilter.checkProfanity(tortureCase.input)), + ); + }); + } + }); + } + + test('legacy path is used when word boundaries are disabled', () => { + const config: FilterConfig = { + languages: ['english'], + wordBoundaries: false, + fuzzyToleranceLevel: 0.6, + }; + const fast = new Filter(config); + const legacy = new Filter({ ...config, disableAhoCorasick: true }); + const text = 'scunthorpe'; + expect(fast.isProfane(text)).toBe(legacy.isProfane(text)); + }); + + test('context-aware mode uses Aho-Corasick when word boundaries are disabled', () => { + const config: FilterConfig = { + languages: ['english'], + enableContextAware: true, + wordBoundaries: false, + fuzzyToleranceLevel: 0.6, + }; + const filter = new Filter(config); + expect(filter['dictionaryMatcher']).toBeTruthy(); + }); + + test('context-aware AC and legacy regex paths agree with word boundaries disabled', () => { + const config: FilterConfig = { + languages: ['english'], + enableContextAware: true, + wordBoundaries: false, + fuzzyToleranceLevel: 0.6, + contextWindow: 3, + confidenceThreshold: 0.7, + }; + const fast = new Filter(config); + const legacy = new Filter({ ...config, disableAhoCorasick: true }); + const cases = [ + 'scunthorpe', + 'This movie is the bomb', + 'You are a fucking idiot', + ]; + + for (const text of cases) { + expect(fast.isProfane(text)).toBe(legacy.isProfane(text)); + expect(fast.checkProfanity(text)).toEqual(legacy.checkProfanity(text)); + } + }); + + test('context-aware mode uses Aho-Corasick for candidate discovery', () => { + const config: FilterConfig = { + languages: ['english'], + enableContextAware: true, + }; + const filter = new Filter(config); + expect(filter['dictionaryMatcher']).toBeTruthy(); + }); + + test('context-aware AC and legacy regex paths agree', () => { + const config: FilterConfig = { + languages: ['english'], + enableContextAware: true, + contextWindow: 3, + confidenceThreshold: 0.7, + }; + const fast = new Filter(config); + const legacy = new Filter({ ...config, disableAhoCorasick: true }); + const cases = [ + 'This movie is the bomb', + 'The bomb exploded and shit happened', + 'You are a fucking idiot', + 'This movie is sick!', + ]; + + for (const text of cases) { + expect(fast.checkProfanity(text)).toEqual(legacy.checkProfanity(text)); + } + }); +}); diff --git a/packages/js/tests/cjk-matching.test.ts b/packages/js/tests/cjk-matching.test.ts new file mode 100644 index 0000000..769df24 --- /dev/null +++ b/packages/js/tests/cjk-matching.test.ts @@ -0,0 +1,137 @@ +import { Filter } from '../src/filters/Filter'; +import type { FilterConfig } from '../src/types/types'; +import { + classifyWordScript, + hasCjkWordBoundary, + hasLatinWordBoundary, + isCjkCharacter, +} from '../src/utils/wordScript'; + +describe('wordScript utilities', () => { + test('isCjkCharacter detects CJK scripts', () => { + expect(isCjkCharacter('你')).toBe(true); + expect(isCjkCharacter('エ')).toBe(true); + expect(isCjkCharacter('병')).toBe(true); + expect(isCjkCharacter('a')).toBe(false); + }); + + test('classifyWordScript uses characters not language', () => { + expect(classifyWordScript('他妈的')).toBe('cjk'); + expect(classifyWordScript('エッチ')).toBe('cjk'); + expect(classifyWordScript('sm')).toBe('latin'); + expect(classifyWordScript('fuck')).toBe('latin'); + }); + + test('hasLatinWordBoundary matches JS \\b semantics', () => { + expect(hasLatinWordBoundary('hello fuck world', 6, 10)).toBe(true); + expect(hasLatinWordBoundary('scunthorpe', 5, 9)).toBe(false); + expect(hasLatinWordBoundary('classic', 2, 5)).toBe(false); + }); + + test('hasCjkWordBoundary allows substring matches including ASCII adjacency', () => { + expect(hasCjkWordBoundary('你他妈的', 1, 4)).toBe(true); + expect(hasCjkWordBoundary('hello操world', 5, 6)).toBe(true); + expect(hasCjkWordBoundary('hello他妈的', 5, 8)).toBe(true); + expect(hasCjkWordBoundary('123エッチ456', 3, 6)).toBe(true); + }); +}); + +describe('CJK automatic matching strategy', () => { + const chineseFilter = new Filter({ languages: ['chinese'] }); + const japaneseFilter = new Filter({ languages: ['japanese'] }); + const koreanFilter = new Filter({ languages: ['korean'] }); + const englishFilter = new Filter({ languages: ['english'] }); + const mixedFilter = new Filter({ languages: ['english', 'chinese'] }); + + describe('detects CJK profanity with default wordBoundaries', () => { + test('Chinese', () => { + expect(chineseFilter.isProfane('你他妈的')).toBe(true); + expect(chineseFilter.checkProfanity('他妈的').profaneWords.length).toBeGreaterThan(0); + }); + + test('Japanese', () => { + expect(japaneseFilter.isProfane('このエッチな話')).toBe(true); + expect(japaneseFilter.isProfane('エッチ')).toBe(true); + }); + + test('Korean', () => { + expect(koreanFilter.isProfane('이 병신')).toBe(true); + expect(koreanFilter.isProfane('병신')).toBe(true); + }); + + test('Japanese ASCII entries still use Latin boundaries', () => { + expect(japaneseFilter.isProfane('hello xx world')).toBe(true); + expect(japaneseFilter.isProfane('xxtra')).toBe(false); + }); + + test('detects CJK profanity wrapped in or adjacent to ASCII', () => { + expect(chineseFilter.isProfane('hello他妈的')).toBe(true); + expect(chineseFilter.isProfane('x乳x')).toBe(true); + expect(chineseFilter.isProfane('123他妈的456')).toBe(true); + expect(japaneseFilter.isProfane('abcエッチdef')).toBe(true); + }); + }); + + describe('preserves English false-positive protection', () => { + test('does not flag scunthorpe or classic for English words', () => { + expect(englishFilter.isProfane('scunthorpe')).toBe(false); + expect(englishFilter.isProfane('classic')).toBe(false); + expect(englishFilter.isProfane('assassin')).toBe(false); + }); + + test('still detects standalone English profanity', () => { + expect(englishFilter.isProfane('hello fuck world')).toBe(true); + }); + + test('mixed filter does not false-positive English traps', () => { + expect(mixedFilter.isProfane('scunthorpe')).toBe(false); + expect(mixedFilter.isProfane('classic')).toBe(false); + }); + }); + + describe('mixed-language filters', () => { + test('detects both English and Chinese in one filter', () => { + expect(mixedFilter.isProfane('hello fuck')).toBe(true); + expect(mixedFilter.isProfane('你他妈的')).toBe(true); + }); + }); + + describe('AC and legacy parity for CJK', () => { + const cases = ['你他妈的', '他妈的', 'エッチ', '병신', 'hello fuck', 'scunthorpe', 'hello他妈的', 'x乳x', 'abcエッチdef']; + + const configs: FilterConfig[] = [ + { languages: ['chinese'] }, + { languages: ['japanese'] }, + { languages: ['korean'] }, + { languages: ['english', 'chinese'] }, + ]; + + for (const config of configs) { + describe(JSON.stringify(config), () => { + const fast = new Filter(config); + const legacy = new Filter({ ...config, disableAhoCorasick: true }); + + for (const text of cases) { + test(`parity for ${JSON.stringify(text)}`, () => { + expect(fast.isProfane(text)).toBe(legacy.isProfane(text)); + expect(fast.checkProfanity(text).containsProfanity).toBe( + legacy.checkProfanity(text).containsProfanity, + ); + }); + } + }); + } + }); + + describe('replacement with CJK words', () => { + test('replaces Chinese profanity without requiring \\b', () => { + const filter = new Filter({ + languages: ['chinese'], + replaceWith: '***', + }); + const result = filter.checkProfanity('你他妈的'); + expect(result.processedText).toContain('***'); + expect(result.processedText).not.toContain('他妈的'); + }); + }); +}); diff --git a/packages/js/tests/context-aware.test.ts b/packages/js/tests/context-aware.test.ts index c6da576..ba077da 100644 --- a/packages/js/tests/context-aware.test.ts +++ b/packages/js/tests/context-aware.test.ts @@ -139,9 +139,20 @@ describe('Context-Aware Filtering', () => { // Should flag without context awareness expect(result.containsProfanity).toBe(true); + expect(traditionalFilter.isProfane('This movie is fucking awesome')).toBe(true); expect(result.matches).toBeUndefined(); expect(result.contextScore).toBeUndefined(); }); + + it('isProfane and checkProfanity agree when context-aware is enabled', () => { + const result = filter.checkProfanity('You are a fucking idiot'); + expect(filter.isProfane('You are a fucking idiot')).toBe(result.containsProfanity); + expect(result.containsProfanity).toBe(true); + + const cleanResult = filter.checkProfanity('This movie is the bomb'); + expect(filter.isProfane('This movie is the bomb')).toBe(cleanResult.containsProfanity); + expect(cleanResult.containsProfanity).toBe(false); + }); }); describe('Configuration Options', () => { diff --git a/packages/js/tests/context-optimization.test.ts b/packages/js/tests/context-optimization.test.ts index f739c82..6d825db 100644 --- a/packages/js/tests/context-optimization.test.ts +++ b/packages/js/tests/context-optimization.test.ts @@ -10,6 +10,16 @@ describe('Context Optimization', () => { }); }); + it('should use Aho-Corasick for candidate discovery', () => { + expect(filter['dictionaryMatcher']).toBeTruthy(); + }); + + it('isProfane applies context filtering', () => { + expect(filter.isProfane('This movie is the bomb')).toBe(false); + expect(filter.isProfane('The bomb exploded and shit happened')).toBe(true); + expect(filter.isProfane('You are a fucking idiot')).toBe(true); + }); + it('should NOT whitelist profanity based on unrelated positive phrases', () => { // "the bomb" is a positive phrase for "bomb". // "shit" is a profanity. diff --git a/packages/js/tests/leetspeak-unicode.test.ts b/packages/js/tests/leetspeak-unicode.test.ts index 126d3bb..0f4531f 100644 --- a/packages/js/tests/leetspeak-unicode.test.ts +++ b/packages/js/tests/leetspeak-unicode.test.ts @@ -15,6 +15,41 @@ import { containsUnicodeObfuscation, detectCharacterSets, } from '../src/utils'; +import { + normalizeEvasion, + collapseSeparatedCharacters, + stripHtmlAndDecodeEntities, +} from '../src/utils/evasion'; + +describe('Evasion Normalization', () => { + describe('stripHtmlAndDecodeEntities', () => { + it('should strip tags and decode numeric entities', () => { + expect(stripHtmlAndDecodeEntities('shit')).toBe('shit'); + expect(stripHtmlAndDecodeEntities('fuck')).toBe('fuck'); + expect(stripHtmlAndDecodeEntities('a
ss')).toBe('ass'); + }); + }); + + describe('collapseSeparatedCharacters', () => { + it('should collapse separator obfuscation', () => { + expect(collapseSeparatedCharacters('f.u.c.k')).toBe('fuck'); + expect(collapseSeparatedCharacters('f_u_c_k')).toBe('fuck'); + expect(collapseSeparatedCharacters('b-i-t-c-h')).toBe('bitch'); + }); + + it('should not collapse normal dotted words', () => { + expect(collapseSeparatedCharacters('hello.world')).toBe('hello.world'); + }); + }); + + describe('normalizeEvasion', () => { + it('should expand masked profanity patterns', () => { + expect(normalizeEvasion('This is f*cking ridiculous')).toBe('This is fucking ridiculous'); + expect(normalizeEvasion('holy f*** that was amazing')).toBe('holy fuck that was amazing'); + expect(normalizeEvasion('go f yourself')).toBe('go fuck yourself'); + }); + }); +}); describe('Leetspeak Detection', () => { describe('normalizeLeetspeak', () => { @@ -95,6 +130,7 @@ describe('Unicode Normalization', () => { // Greek letters that look like Latin expect(normalizeUnicode('fυck')).toBe('fuck'); // Greek upsilon expect(normalizeUnicode('fосk')).toBe('fock'); // Cyrillic о + expect(normalizeUnicode('fսck')).toBe('fuck'); // Armenian seh }); }); @@ -215,6 +251,7 @@ describe('Filter with Leetspeak and Unicode', () => { const filter = new Filter({ languages: ['english'], detectLeetspeak: true, + leetspeakLevel: 'aggressive', normalizeUnicode: true, fuzzyToleranceLevel: 0.7, }); @@ -225,6 +262,23 @@ describe('Filter with Leetspeak and Unicode', () => { expect(filter.isProfane('f\u200Buck')).toBe(true); // zero-width in "fuck" expect(filter.isProfane('sh!t')).toBe(true); }); + + it('should detect shootout evasion categories', () => { + const cases = [ + 'shi7e', + 'f.u.c.k', + 'a
ss', + 'shit', + 'This is f*cking ridiculous', + 'holy f*** that was amazing', + 'go f yourself', + 'what a f@cking mess', + 'fսck', + ]; + for (const text of cases) { + expect(filter.isProfane(text)).toBe(true); + } + }); }); describe('Caching', () => { diff --git a/packages/js/tests/useProfanityChecker.test.tsx b/packages/js/tests/useProfanityChecker.test.tsx index a55a4ac..37b27fd 100644 --- a/packages/js/tests/useProfanityChecker.test.tsx +++ b/packages/js/tests/useProfanityChecker.test.tsx @@ -1,14 +1,23 @@ import { renderHook, act } from '@testing-library/react'; +const mockFilterIsProfane = jest.fn(); + // Mock the core functions to ensure we're testing the hook behavior jest.mock('../src/core', () => ({ checkProfanity: jest.fn(), checkProfanityAsync: jest.fn(), - isWordProfane: jest.fn(), +})); + +jest.mock('../src/core/filterPool', () => ({ + createFilterConfig: jest.fn((config?: unknown) => config ?? {}), + getPooledFilter: jest.fn(() => ({ + isProfane: mockFilterIsProfane, + })), })); // Mock React hooks jest.mock('react', () => ({ + ...jest.requireActual('react'), useState: jest.fn(), useCallback: jest.fn(), })); @@ -19,7 +28,6 @@ import { SeverityLevel, Language } from '../src/types/types'; // Get the mocked functions const mockCheckProfanity = jest.mocked(require('../src/core').checkProfanity); const mockCheckProfanityAsync = jest.mocked(require('../src/core').checkProfanityAsync); -const mockIsWordProfane = jest.mocked(require('../src/core').isWordProfane); const mockUseState = jest.mocked(require('react').useState); const mockUseCallback = jest.mocked(require('react').useCallback); @@ -112,16 +120,16 @@ describe('useProfanityChecker Hook', () => { }); describe('Word Checking', () => { - test('calls isWordProfane with correct parameters', () => { - mockIsWordProfane.mockReturnValue(true); - + test('calls filter.isProfane for single-word checks', () => { + mockFilterIsProfane.mockReturnValue(true); + const { result } = renderHook(() => useProfanityChecker()); - + act(() => { result.current.isWordProfane('badword'); }); - expect(mockIsWordProfane).toHaveBeenCalledWith('badword', undefined); + expect(mockFilterIsProfane).toHaveBeenCalledWith('badword'); }); }); diff --git a/packages/py/glin_profanity/core/__init__.py b/packages/py/glin_profanity/core/__init__.py new file mode 100644 index 0000000..ecfd952 --- /dev/null +++ b/packages/py/glin_profanity/core/__init__.py @@ -0,0 +1,15 @@ +"""Core helpers for glin-profanity.""" + +from glin_profanity.core.filter_pool import ( + clear_filter_pool, + config_cache_key, + create_filter_config, + get_pooled_filter, +) + +__all__ = [ + "clear_filter_pool", + "config_cache_key", + "create_filter_config", + "get_pooled_filter", +] diff --git a/packages/py/glin_profanity/core/filter_pool.py b/packages/py/glin_profanity/core/filter_pool.py new file mode 100644 index 0000000..6af1318 --- /dev/null +++ b/packages/py/glin_profanity/core/filter_pool.py @@ -0,0 +1,96 @@ +"""Shared Filter instance pool (mirrors packages/js/src/core/filterPool.ts).""" + +from __future__ import annotations + +import json +import warnings +from collections import OrderedDict +from pathlib import Path +from typing import Any + +from glin_profanity.filters.filter import Filter +from glin_profanity.types.types import FilterConfig + +FILTER_POOL_MAX = 32 +_filter_pool: OrderedDict[str, Filter] = OrderedDict() + +_GLOBAL_WHITELIST_PATH = ( + Path(__file__).resolve().parent.parent + / "data" + / "dictionaries" + / "globalWhitelist.json" +) + + +def _load_global_whitelist() -> list[str]: + with _GLOBAL_WHITELIST_PATH.open(encoding="utf-8") as handle: + data = json.load(handle) + return list(data.get("whitelist", [])) + + +def create_filter_config(config: FilterConfig | None = None) -> FilterConfig: + """Build effective filter config, merging the shared global whitelist.""" + effective: dict[str, Any] = dict(config or {}) + user_ignore = effective.get("ignore_words") or [] + effective["ignore_words"] = [*_load_global_whitelist(), *user_ignore] + effective.setdefault("fuzzy_tolerance_level", 0.8) + + if effective.get("allow_obfuscated_match") and effective.get("word_boundaries", True): + warnings.warn( + "[Glin-Profanity] Obfuscated match enabled → wordBoundaries will be ignored internally.", + stacklevel=2, + ) + + return effective # type: ignore[return-value] + + +def _normalize_config_for_key(config: FilterConfig) -> FilterConfig: + normalized: dict[str, Any] = dict(config) + + languages = normalized.get("languages") + if languages: + normalized["languages"] = sorted(languages) + + ignore_words = normalized.get("ignore_words") + if ignore_words: + normalized["ignore_words"] = sorted(ignore_words) + + custom_words = normalized.get("custom_words") + if custom_words: + normalized["custom_words"] = sorted(custom_words) + + domain_whitelists = normalized.get("domain_whitelists") + if domain_whitelists: + normalized["domain_whitelists"] = { + lang: sorted(domain_whitelists[lang]) + for lang in sorted(domain_whitelists) + } + + return normalized # type: ignore[return-value] + + +def config_cache_key(config: FilterConfig) -> str: + return json.dumps(_normalize_config_for_key(config), sort_keys=True) + + +def get_pooled_filter(config: FilterConfig | None = None) -> Filter: + """Return a shared Filter for the given configuration (FIFO eviction at max size).""" + effective = create_filter_config(config) + key = config_cache_key(effective) + + existing = _filter_pool.get(key) + if existing is not None: + return existing + + filter_instance = Filter(effective) + if len(_filter_pool) >= FILTER_POOL_MAX: + oldest_key = next(iter(_filter_pool)) + del _filter_pool[oldest_key] + + _filter_pool[key] = filter_instance + return filter_instance + + +def clear_filter_pool() -> None: + """Clear all pooled Filter instances (intended for tests).""" + _filter_pool.clear() diff --git a/packages/py/glin_profanity/data/dictionaries/english.json b/packages/py/glin_profanity/data/dictionaries/english.json index a8aecf6..1ce990e 100644 --- a/packages/py/glin_profanity/data/dictionaries/english.json +++ b/packages/py/glin_profanity/data/dictionaries/english.json @@ -329,6 +329,7 @@ "shemale", "shibari", "shit", + "shite", "sh1t", "$hit", "$h!t", diff --git a/packages/py/glin_profanity/filters/dictionary_aho_corasick.py b/packages/py/glin_profanity/filters/dictionary_aho_corasick.py new file mode 100644 index 0000000..cfd0166 --- /dev/null +++ b/packages/py/glin_profanity/filters/dictionary_aho_corasick.py @@ -0,0 +1,145 @@ +"""Multi-pattern dictionary matcher backed by the Aho-Corasick algorithm.""" + +from __future__ import annotations + +import unicodedata +from dataclasses import dataclass +from typing import Iterable + +import ahocorasick + +from glin_profanity.utils.word_script import WordScript, match_has_word_boundary + + +@dataclass(frozen=True) +class DictionaryMatch: + dict_word: str + start: int + end: int + matched_text: str + + +@dataclass(frozen=True) +class DictionarySearchOptions: + word_boundaries: bool + case_sensitive: bool + ignore_words: set[str] + word_scripts: dict[str, WordScript] + + +def _count_graphemes(text: str) -> int: + return sum(1 for _ in _segment_graphemes(text)) + + +def _segment_graphemes(text: str) -> Iterable[tuple[int, str]]: + """Yield (string_index, segment) for each extended grapheme cluster.""" + if not text: + return + + index = 0 + length = len(text) + while index < length: + start = index + index += 1 + while index < length and _is_grapheme_extend(text, index): + index += 1 + yield start, text[start:index] + + +def _is_grapheme_extend(text: str, index: int) -> bool: + char = text[index] + category = unicodedata.category(char) + if category in {"Mn", "Me", "Mc"}: + return True + if char == "\u200d": # ZWJ + return True + return category == "Cf" and char in { + "\u200c", # ZWNJ + "\u200d", # ZWJ + "\ufe0e", + "\ufe0f", + } + + +def _grapheme_start_to_string_index(text: str, grapheme_index: int) -> int: + for i, (start, _segment) in enumerate(_segment_graphemes(text)): + if i == grapheme_index: + return start + return len(text) + + +def _grapheme_end_to_exclusive_string_index( + text: str, end_grapheme_index_inclusive: int +) -> int: + for i, (start, segment) in enumerate(_segment_graphemes(text)): + if i == end_grapheme_index_inclusive: + return start + len(segment) + return len(text) + + +def _code_point_end_to_grapheme_end(text: str, end_index_inclusive: int) -> int: + grapheme_end = -1 + for i, (start, segment) in enumerate(_segment_graphemes(text)): + segment_end = start + len(segment) - 1 + if start <= end_index_inclusive <= segment_end: + return i + if end_index_inclusive < start: + break + grapheme_end = i + return max(grapheme_end, 0) + + +class DictionaryAhoCorasick: + """Exact dictionary matching when word boundaries are enabled (no fuzzy path).""" + + def __init__(self, words: list[str]) -> None: + self._automaton = ahocorasick.Automaton() + for word in words: + if word: + self._automaton.add_word(word, word) + self._automaton.make_automaton() + self._word_grapheme_lengths = { + word: _count_graphemes(word) for word in words if word + } + + def has_any_match(self, text: str, options: DictionarySearchOptions) -> bool: + return bool(self.find_matches(text, options)) + + def find_matches( + self, text: str, options: DictionarySearchOptions + ) -> list[DictionaryMatch]: + haystack = text if options.case_sensitive else text.lower() + results: list[DictionaryMatch] = [] + seen: set[str] = set() + + for end_index_inclusive, dict_word in self._automaton.iter(haystack): + if dict_word.lower() in options.ignore_words: + continue + + word_len = self._word_grapheme_lengths.get(dict_word, _count_graphemes(dict_word)) + end_grapheme = _code_point_end_to_grapheme_end(haystack, end_index_inclusive) + start_grapheme = end_grapheme - word_len + 1 + start = _grapheme_start_to_string_index(text, start_grapheme) + end = _grapheme_end_to_exclusive_string_index(text, end_grapheme) + script = options.word_scripts.get(dict_word, "latin") + + if not match_has_word_boundary( + text, start, end, script, options.word_boundaries + ): + continue + + dedupe_key = f"{dict_word}:{start}:{end}" + if dedupe_key in seen: + continue + seen.add(dedupe_key) + + results.append( + DictionaryMatch( + dict_word=dict_word, + start=start, + end=end, + matched_text=text[start:end], + ) + ) + + return results diff --git a/packages/py/glin_profanity/filters/filter.py b/packages/py/glin_profanity/filters/filter.py index 2ecdca7..5bd6902 100644 --- a/packages/py/glin_profanity/filters/filter.py +++ b/packages/py/glin_profanity/filters/filter.py @@ -6,14 +6,30 @@ from typing import Literal from glin_profanity.data.dictionary import dictionary +from glin_profanity.filters.dictionary_aho_corasick import ( + DictionaryAhoCorasick, + DictionaryMatch, + DictionarySearchOptions, +) +from glin_profanity.nlp.context_analyzer import ContextAnalyzer, ContextConfig from glin_profanity.types.types import ( CheckProfanityResult, FilterConfig, + Language, Match, SeverityLevel, ) -from glin_profanity.utils.leetspeak import normalize_leetspeak +from glin_profanity.utils.evasion import normalize_evasion +from glin_profanity.utils.leetspeak import ( + normalize_leetspeak, + normalize_leetspeak_variants, +) from glin_profanity.utils.unicode import normalize_unicode +from glin_profanity.utils.word_script import ( + WordScript, + classify_word_script, + match_has_word_boundary, +) LeetspeakLevel = Literal["basic", "moderate", "aggressive"] @@ -58,6 +74,18 @@ def __init__(self, config: FilterConfig | None = None) -> None: self.enable_context_aware = config.get("enable_context_aware", False) self.context_window = config.get("context_window", 3) self.confidence_threshold = config.get("confidence_threshold", 0.7) + languages = config.get("languages", ["english"]) + self.primary_language: Language = languages[0] if languages else "english" + self.context_analyzer: ContextAnalyzer | None = None + if self.enable_context_aware: + domain_whitelists = config.get("domain_whitelists") or {} + self.context_analyzer = ContextAnalyzer( + ContextConfig( + context_window=self.context_window, + language=self.primary_language, + domain_whitelists=domain_whitelists.get(self.primary_language, []), + ) + ) # Leetspeak and Unicode normalization configuration self.detect_leetspeak = config.get("detect_leetspeak", False) @@ -69,6 +97,7 @@ def __init__(self, config: FilterConfig | None = None) -> None: self.max_cache_size = config.get("max_cache_size", 1000) self._cache: dict[str, CheckProfanityResult] = {} self._regex_cache: dict[str, re.Pattern[str]] = {} + self.dictionary_matcher: DictionaryAhoCorasick | None = None # Initialize word sets ignore_words_list = config.get("ignore_words", []) @@ -94,14 +123,55 @@ def _load_words(self, config: FilterConfig) -> None: if custom_words: words.extend(custom_words) - # Store as set for faster lookup + # Store as set for faster lookup; track script per entry for boundary rules self.words: set[str] = {word.lower() for word in words} + self.word_scripts: dict[str, WordScript] = {} + for word in words: + key = word.lower() + self.word_scripts[key] = classify_word_script(word) + + if self._should_use_aho_corasick(config): + self.dictionary_matcher = DictionaryAhoCorasick(list(self.words)) + + def _should_use_aho_corasick(self, config: FilterConfig) -> bool: + if config.get("disable_aho_corasick"): + return False + return self.word_boundaries or self.enable_context_aware + + def _get_dictionary_search_options(self) -> DictionarySearchOptions: + return DictionarySearchOptions( + word_boundaries=self.word_boundaries, + case_sensitive=self.case_sensitive, + ignore_words=self.ignore_words, + word_scripts=self.word_scripts, + ) def _debug_log(self, *args: object) -> None: """Log debug information if logging is enabled.""" if self.log_profanity: print("[glin-profanity]", *args) # noqa: T201 + def _get_normalized_variants(self, text: str) -> tuple[str, str]: + """Compute normal and aggressive normalized text in one pass.""" + base = normalize_evasion(text) + + if self.normalize_unicode_enabled: + base = normalize_unicode(base) + + if self.detect_leetspeak: + return normalize_leetspeak_variants( + base, + level=self.leetspeak_level, + collapse_repeated=True, + remove_spaced_chars=True, + ) + + if self.allow_obfuscated_match and not self.detect_leetspeak: + obfuscated = self._normalize_obfuscated(base) + return obfuscated, obfuscated + + return base, base + def _normalize_text(self, text: str) -> str: """ Normalize text for profanity detection using all enabled normalization methods. @@ -114,26 +184,76 @@ def _normalize_text(self, text: str) -> str: Returns: The normalized text """ - normalized = text + normal, _ = self._get_normalized_variants(text) + return normal + + def _get_text_variants(self, text: str, lowercase: bool) -> dict[str, str]: + normal, aggressive = self._get_normalized_variants(text) + if lowercase: + return { + "original": text.lower(), + "normalized": normal.lower(), + "aggressive": aggressive.lower(), + } + return { + "original": text, + "normalized": normal, + "aggressive": aggressive, + } - # Step 1: Apply Unicode normalization (handles homoglyphs, diacritics, etc.) - if self.normalize_unicode_enabled: - normalized = normalize_unicode(normalized) + def _evaluate_severity_on_variants( + self, word: str, variants: dict[str, str] + ) -> SeverityLevel | None: + severity = self._evaluate_severity(word, variants["original"]) + if severity is not None: + return severity - # Step 2: Apply leetspeak normalization - if self.detect_leetspeak: - normalized = normalize_leetspeak( - normalized, - level=self.leetspeak_level, - collapse_repeated=True, - remove_spaced_chars=True, - ) + if variants["normalized"] != variants["original"]: + severity = self._evaluate_severity(word, variants["normalized"]) + if severity is not None: + return severity - # Step 3: Apply legacy obfuscation handling (for backward compatibility) - if self.allow_obfuscated_match and not self.detect_leetspeak: - normalized = self._normalize_obfuscated(normalized) + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + return self._evaluate_severity(word, variants["aggressive"]) - return normalized + return None + + def _collect_matches_from_variant( + self, + dict_word: str, + variant_text: str, + severity: SeverityLevel, + profane_words: set[str], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + use_match_text: bool, + ) -> None: + regex = self._get_regex(dict_word) + script = self._get_word_script(dict_word) + + for match in regex.finditer(variant_text): + start = match.start() + end = match.end() + if not match_has_word_boundary( + variant_text, start, end, script, self.word_boundaries + ): + continue + + matched = match.group(0) if use_match_text else dict_word + profane_words.add(matched) + if matched not in severity_map: + severity_map[matched] = severity + + matches.append( + { + "word": matched, + "index": start, + "severity": severity, + } + ) def _normalize_obfuscated(self, text: str) -> str: """ @@ -233,20 +353,34 @@ def _get_from_cache(self, key: str) -> CheckProfanityResult | None: return None return self._cache.get(key) + def _get_word_script(self, word: str) -> WordScript: + return self.word_scripts.get(word, classify_word_script(word)) + def _get_regex(self, word: str) -> re.Pattern[str]: """Create regex pattern for word matching.""" - if word in self._regex_cache: - return self._regex_cache[word] + script = self._get_word_script(word) + cache_key = f"{script}:{word}" + if cache_key in self._regex_cache: + return self._regex_cache[cache_key] flags = 0 if self.case_sensitive else re.IGNORECASE escaped_word = re.escape(word) - - pattern = rf"\b{escaped_word}\b" if self.word_boundaries else escaped_word + use_latin_boundary = self.word_boundaries and script == "latin" + boundary = r"\b" if use_latin_boundary else "" + pattern = f"{boundary}{escaped_word}{boundary}" regex = re.compile(pattern, flags) - self._regex_cache[word] = regex + self._regex_cache[cache_key] = regex return regex + def _get_replacement_regex(self, word: str) -> re.Pattern[str]: + escaped = re.escape(word) + if not self.word_boundaries: + return re.compile(escaped, re.IGNORECASE) + if classify_word_script(word) == "cjk": + return re.compile(escaped, re.IGNORECASE) + return re.compile(rf"\b{escaped}\b", re.IGNORECASE) + def _is_fuzzy_tolerance_match(self, word: str, text: str) -> bool: """Check if word matches text within fuzzy tolerance.""" simplified_text = re.sub(r"[^a-z]", "", text.lower()) @@ -294,15 +428,450 @@ def _fuzzy_match_single_word(self, pattern_word: str, text_word: str) -> bool: def _evaluate_severity(self, word: str, text: str) -> SeverityLevel | None: """Evaluate the severity level of a match.""" + script = self._get_word_script(word) regex = self._get_regex(word) + if script == "cjk" and self.word_boundaries: + for match in regex.finditer(text): + start = match.start() + end = match.end() + if match_has_word_boundary(text, start, end, script, True): + return SeverityLevel.EXACT + return None + if regex.search(text): return SeverityLevel.EXACT - if self._is_fuzzy_tolerance_match(word, text): + if not self.word_boundaries and self._is_fuzzy_tolerance_match(word, text): return SeverityLevel.FUZZY return None + def _is_profane_with_aho_corasick(self, value: str) -> bool: + variants = self._get_text_variants(value, False) + options = self._get_dictionary_search_options() + matcher = self.dictionary_matcher + assert matcher is not None + + if matcher.has_any_match(variants["original"], options): + return True + if variants["normalized"] != variants["original"] and matcher.has_any_match( + variants["normalized"], options + ): + return True + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + and matcher.has_any_match(variants["aggressive"], options) + ): + return True + return False + + def _is_profane_legacy(self, value: str) -> bool: + variants = self._get_text_variants(value, False) + + for word in self.words: + if word.lower() in self.ignore_words: + continue + if self._evaluate_severity_on_variants(word, variants) is not None: + return True + + return False + + def _check_profanity_with_aho_corasick(self, text: str) -> CheckProfanityResult: + variants = self._get_text_variants(text, True) + profane_words: set[str] = set() + severity_map: dict[str, SeverityLevel] = {} + options = self._get_dictionary_search_options() + matcher = self.dictionary_matcher + assert matcher is not None + + for match in matcher.find_matches(variants["original"], options): + profane_words.add(match.matched_text) + if match.matched_text not in severity_map: + severity_map[match.matched_text] = SeverityLevel.EXACT + + if variants["normalized"] != variants["original"]: + for match in matcher.find_matches(variants["normalized"], options): + profane_words.add(match.dict_word) + if match.dict_word not in severity_map: + severity_map[match.dict_word] = SeverityLevel.EXACT + + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + for match in matcher.find_matches(variants["aggressive"], options): + profane_words.add(match.dict_word) + if match.dict_word not in severity_map: + severity_map[match.dict_word] = SeverityLevel.EXACT + + return self._build_profanity_result(text, profane_words, severity_map) + + def _check_profanity_legacy_non_context(self, text: str) -> CheckProfanityResult: + variants = self._get_text_variants(text, True) + profane_words_set: set[str] = set() + severity_map: dict[str, SeverityLevel] = {} + matches: list[Match] = [] + + for dict_word in self.words: + if dict_word.lower() in self.ignore_words: + continue + + severity = self._evaluate_severity(dict_word, variants["original"]) + if severity is not None: + self._collect_matches_from_variant( + dict_word, + variants["original"], + severity, + profane_words_set, + severity_map, + matches, + True, + ) + + if variants["normalized"] != variants["original"]: + severity = self._evaluate_severity(dict_word, variants["normalized"]) + if severity is not None: + self._collect_matches_from_variant( + dict_word, + variants["normalized"], + severity, + profane_words_set, + severity_map, + matches, + False, + ) + + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + severity = self._evaluate_severity(dict_word, variants["aggressive"]) + if severity is not None: + profane_words_set.add(dict_word) + if dict_word not in severity_map: + severity_map[dict_word] = SeverityLevel.EXACT + + result = self._build_profanity_result(text, profane_words_set, severity_map) + if matches: + result["matches"] = matches + result["reason"] = ( + f"Found {len(matches)} potential profanity matches" + if matches + else "No profanity detected" + ) + return result + + def _build_profanity_result( + self, + text: str, + profane_words: set[str], + severity_map: dict[str, SeverityLevel], + ) -> CheckProfanityResult: + profane_word_list = list(profane_words) + processed_text = text + + if self.replace_with and profane_word_list: + for word in profane_word_list: + replacement_regex = self._get_replacement_regex(word) + processed_text = replacement_regex.sub(self.replace_with, processed_text) + + result: CheckProfanityResult = { + "contains_profanity": len(profane_word_list) > 0, + "profane_words": profane_word_list, + "reason": ( + f"Found {len(profane_word_list)} potential profanity matches" + if profane_word_list + else "No profanity detected" + ), + } + + if self.replace_with: + result["processed_text"] = processed_text + + if self.severity_levels and severity_map: + result["severity_map"] = severity_map + + return result + + def _passes_context_filter( + self, text: str, matched_word: str, match_index: int + ) -> bool: + if not self.context_analyzer: + return True + context_result = self.context_analyzer.analyze_context( + text, matched_word, match_index + ) + return not ( + context_result.is_whitelisted + or context_result.context_score > self.confidence_threshold + ) + + def _record_context_aware_match( + self, + text: str, + matched_word: str, + match_index: int, + severity: SeverityLevel, + profane_words: list[str], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + seen: set[str], + ) -> None: + dedupe_key = f"{matched_word}:{match_index}" + if dedupe_key in seen: + return + + match_obj: Match = { + "word": matched_word, + "index": match_index, + "severity": severity, + } + + if self.context_analyzer: + context_result = self.context_analyzer.analyze_context( + text, matched_word, match_index + ) + match_obj["context_score"] = context_result.context_score + match_obj["reason"] = context_result.reason + match_obj["is_whitelisted"] = context_result.is_whitelisted + if not self._passes_context_filter(text, matched_word, match_index): + return + + seen.add(dedupe_key) + profane_words.append(matched_word) + if matched_word not in severity_map: + severity_map[matched_word] = severity + matches.append(match_obj) + + def _collect_context_aware_candidates_from_ac( + self, + text: str, + variants: dict[str, str], + profane_words: list[str], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + seen: set[str], + ) -> None: + options = self._get_dictionary_search_options() + matcher = self.dictionary_matcher + assert matcher is not None + + def process_ac_match(match: DictionaryMatch, use_matched_text: bool) -> None: + self._record_context_aware_match( + text, + match.matched_text if use_matched_text else match.dict_word, + match.start, + SeverityLevel.EXACT, + profane_words, + severity_map, + matches, + seen, + ) + + for match in matcher.find_matches(variants["original"], options): + process_ac_match(match, True) + + if variants["normalized"] != variants["original"]: + for match in matcher.find_matches(variants["normalized"], options): + process_ac_match(match, False) + + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + for match in matcher.find_matches(variants["aggressive"], options): + process_ac_match(match, False) + + def _collect_context_aware_candidates_from_legacy( + self, + text: str, + variants: dict[str, str], + profane_words: list[str], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + seen: set[str], + ) -> None: + for dict_word in self.words: + if dict_word.lower() in self.ignore_words: + continue + + def collect_from_variant(variant_text: str, use_match_text: bool) -> None: + severity = self._evaluate_severity(dict_word, variant_text) + if severity is None: + return + + regex = self._get_regex(dict_word) + script = self._get_word_script(dict_word) + for match in regex.finditer(variant_text): + start = match.start() + end = match.end() + if not match_has_word_boundary( + variant_text, start, end, script, self.word_boundaries + ): + continue + self._record_context_aware_match( + text, + match.group(0) if use_match_text else dict_word, + start, + severity, + profane_words, + severity_map, + matches, + seen, + ) + + collect_from_variant(variants["original"], True) + + if variants["normalized"] != variants["original"]: + collect_from_variant(variants["normalized"], False) + + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + collect_from_variant(variants["aggressive"], False) + + def _build_context_aware_result( + self, + text: str, + profane_words: list[str], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + ) -> CheckProfanityResult: + processed_text = text + if self.replace_with and profane_words: + for word in dict.fromkeys(profane_words): + processed_text = self._get_replacement_regex(word).sub( + self.replace_with, processed_text + ) + + context_score: float | None = None + if matches: + context_score = sum( + match.get("context_score") or 0.5 for match in matches + ) / len(matches) + + result: CheckProfanityResult = { + "contains_profanity": len(profane_words) > 0, + "profane_words": list(dict.fromkeys(profane_words)), + "reason": ( + f"Found {len(matches)} potential profanity matches" + if matches + else "No profanity detected" + ), + } + + if self.replace_with: + result["processed_text"] = processed_text + if self.severity_levels and severity_map: + result["severity_map"] = severity_map + if matches: + result["matches"] = matches + if context_score is not None: + result["context_score"] = context_score + + return result + + def _check_profanity_with_context_aware(self, text: str) -> CheckProfanityResult: + variants = self._get_text_variants(text, True) + profane_words: list[str] = [] + severity_map: dict[str, SeverityLevel] = {} + matches: list[Match] = [] + seen: set[str] = set() + + if self.dictionary_matcher: + self._collect_context_aware_candidates_from_ac( + text, variants, profane_words, severity_map, matches, seen + ) + else: + self._collect_context_aware_candidates_from_legacy( + text, variants, profane_words, severity_map, matches, seen + ) + + if profane_words: + self._debug_log("Detected:", profane_words) + + return self._build_context_aware_result( + text, profane_words, severity_map, matches + ) + + def _has_context_aware_ac_match(self, text: str, variants: dict[str, str]) -> bool: + options = self._get_dictionary_search_options() + matcher = self.dictionary_matcher + assert matcher is not None + + def has_flagged_match(variant_text: str, use_matched_text: bool) -> bool: + for match in matcher.find_matches(variant_text, options): + matched_word = match.matched_text if use_matched_text else match.dict_word + if self._passes_context_filter(text, matched_word, match.start): + return True + return False + + if has_flagged_match(variants["original"], True): + return True + if variants["normalized"] != variants["original"] and has_flagged_match( + variants["normalized"], False + ): + return True + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + and has_flagged_match(variants["aggressive"], False) + ): + return True + return False + + def _has_context_aware_legacy_match_for_word( + self, text: str, variants: dict[str, str], dict_word: str + ) -> bool: + def check_variant(variant_text: str, use_match_text: bool) -> bool: + severity = self._evaluate_severity(dict_word, variant_text) + if severity is None: + return False + + regex = self._get_regex(dict_word) + script = self._get_word_script(dict_word) + for match in regex.finditer(variant_text): + start = match.start() + end = match.end() + if not match_has_word_boundary( + variant_text, start, end, script, self.word_boundaries + ): + continue + matched_word = match.group(0) if use_match_text else dict_word + if self._passes_context_filter(text, matched_word, start): + return True + return False + + if check_variant(variants["original"], True): + return True + if variants["normalized"] != variants["original"] and check_variant( + variants["normalized"], False + ): + return True + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + and check_variant(variants["aggressive"], False) + ): + return True + return False + + def _has_context_aware_legacy_match(self, text: str, variants: dict[str, str]) -> bool: + for dict_word in self.words: + if dict_word.lower() in self.ignore_words: + continue + if self._has_context_aware_legacy_match_for_word(text, variants, dict_word): + return True + return False + + def _is_profane_with_context_aware(self, value: str) -> bool: + variants = self._get_text_variants(value, False) + if self.dictionary_matcher: + return self._has_context_aware_ac_match(value, variants) + return self._has_context_aware_legacy_match(value, variants) + def is_profane(self, value: str) -> bool: """ Check if text contains profanity. @@ -322,17 +891,11 @@ def is_profane(self, value: str) -> bool: >>> filter.is_profane("f u c k") True """ - # Apply all normalizations - input_text = self._normalize_text(value) - - for word in self.words: - if ( - word.lower() not in self.ignore_words - and self._evaluate_severity(word, input_text) is not None - ): - return True - - return False + if self.enable_context_aware: + return self._is_profane_with_context_aware(value) + if self.dictionary_matcher: + return self._is_profane_with_aho_corasick(value) + return self._is_profane_legacy(value) def matches(self, word: str) -> bool: """ @@ -368,83 +931,19 @@ def check_profanity(self, text: str) -> CheckProfanityResult: self._debug_log("Cache hit for:", text[:50]) return cached_result - # Apply all normalizations - input_text = self._normalize_text(text) - input_lower = input_text.lower() - - profane_words: list[str] = [] - severity_map: dict[str, SeverityLevel] = {} - matches: list[Match] = [] - - # Check each word in dictionary - for dict_word in self.words: - if dict_word.lower() in self.ignore_words: - continue - - severity = self._evaluate_severity(dict_word, input_text) - if severity is not None: - regex = self._get_regex(dict_word) - - # Find all matches - for match in regex.finditer(input_text): - matched_word = match.group(0) - match_index = match.start() - - profane_words.append(matched_word) - severity_map[matched_word] = severity - - # Create match object - match_obj: Match = { - "word": matched_word, - "index": match_index, - "severity": severity, - } - - # TODO: Add context analysis when implemented - matches.append(match_obj) - - # Log detected profanity - if profane_words: - self._debug_log("Detected:", profane_words) - - # Process text replacement if configured - processed_text = text - if self.replace_with and profane_words: - unique_words = list(set(profane_words)) - for word in unique_words: - escaped = re.escape(word) - if self.word_boundaries: - replacement_regex = re.compile(rf"\b{escaped}\b", re.IGNORECASE) - else: - replacement_regex = re.compile(escaped, re.IGNORECASE) - processed_text = replacement_regex.sub( - self.replace_with, processed_text - ) - - # Build result - result: CheckProfanityResult = { - "contains_profanity": len(profane_words) > 0, - "profane_words": list(set(profane_words)), - } - - if self.replace_with: - result["processed_text"] = processed_text - - if self.severity_levels and severity_map: - result["severity_map"] = severity_map - - if matches: - result["matches"] = matches - - result["reason"] = ( - f"Found {len(matches)} potential profanity matches" - if matches - else "No profanity detected" - ) + if not self.enable_context_aware: + result = ( + self._check_profanity_with_aho_corasick(text) + if self.dictionary_matcher + else self._check_profanity_legacy_non_context(text) + ) + if result["contains_profanity"]: + self._debug_log("Detected:", result.get("profane_words", [])) + self._add_to_cache(text, result) + return result - # Cache the result + result = self._check_profanity_with_context_aware(text) self._add_to_cache(text, result) - return result def check_profanity_with_min_severity( diff --git a/packages/py/glin_profanity/nlp/__init__.py b/packages/py/glin_profanity/nlp/__init__.py index 2425b34..2ab21d4 100644 --- a/packages/py/glin_profanity/nlp/__init__.py +++ b/packages/py/glin_profanity/nlp/__init__.py @@ -1 +1,9 @@ -"""NLP components for glin-profanity.""" +"""ContextAnalyzer exports.""" + +from glin_profanity.nlp.context_analyzer import ( + ContextAnalysisResult, + ContextAnalyzer, + ContextConfig, +) + +__all__ = ["ContextAnalysisResult", "ContextAnalyzer", "ContextConfig"] diff --git a/packages/py/glin_profanity/nlp/context_analyzer.py b/packages/py/glin_profanity/nlp/context_analyzer.py new file mode 100644 index 0000000..0a69cfd --- /dev/null +++ b/packages/py/glin_profanity/nlp/context_analyzer.py @@ -0,0 +1,369 @@ +"""Context analysis for distinguishing profanity from false positives.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Literal + +Language = Literal[ + "arabic", + "chinese", + "czech", + "danish", + "dutch", + "english", + "esperanto", + "finnish", + "french", + "german", + "hindi", + "hungarian", + "italian", + "japanese", + "korean", + "norwegian", + "persian", + "polish", + "portuguese", + "russian", + "spanish", + "swedish", + "thai", + "turkish", +] + + +@dataclass(frozen=True) +class ContextAnalysisResult: + context_score: float + reason: str + is_whitelisted: bool + + +@dataclass +class ContextConfig: + context_window: int + language: Language + domain_whitelists: list[str] | None = None + + +POSITIVE_INDICATORS = frozenset( + { + "amazing", + "awesome", + "excellent", + "fantastic", + "great", + "love", + "wonderful", + "brilliant", + "perfect", + "incredible", + "outstanding", + "superb", + "magnificent", + "marvelous", + "spectacular", + "phenomenal", + "terrific", + "fabulous", + "divine", + "best", + "good", + "nice", + "cool", + "sweet", + "rad", + "sick", + "dope", + "fire", + "lit", + "epic", + "legendary", + "godlike", + "insane", + "crazy", + "wild", + "beast", + "movie", + "film", + "show", + "song", + "music", + "game", + "book", + "restaurant", + "food", + "dish", + "meal", + "place", + "spot", + "location", + "experience", + } +) + +NEGATIVE_INDICATORS = frozenset( + { + "hate", + "terrible", + "awful", + "horrible", + "disgusting", + "pathetic", + "stupid", + "idiot", + "moron", + "loser", + "worthless", + "useless", + "garbage", + "trash", + "suck", + "sucks", + "worst", + "bad", + "ugly", + "gross", + "nasty", + "annoying", + "irritating", + "frustrating", + "disappointing", + "lame", + "weak", + "fail", + "you", + "your", + "yourself", + "u", + "ur", + "ure", + "youre", + } +) + +GAMING_POSITIVE = frozenset( + { + "player", + "gamer", + "team", + "squad", + "clan", + "guild", + "match", + "game", + "round", + "level", + "boss", + "raid", + "quest", + "achievement", + "skill", + "build", + "loadout", + "strategy", + "tactic", + "play", + "move", + "combo", + } +) + +GAMING_ACCEPTABLE_WORDS = frozenset( + { + "kill", + "killer", + "killed", + "killing", + "shoot", + "shot", + "shooting", + "die", + "dying", + "died", + "dead", + "death", + "badass", + "sick", + "insane", + "crazy", + "mad", + "beast", + "savage", + "suck", + "sucks", + "wtf", + "omg", + "hell", + "damn", + "crap", + } +) + +POSITIVE_PHRASES: dict[str, float] = { + "the bomb": 0.9, + "da bomb": 0.9, + "bomb.com": 0.9, + "bomb diggity": 0.9, + "photo bomb": 0.8, + "bath bomb": 0.8, + "bomb squad": 0.7, +} + +NEGATIVE_PHRASES: dict[str, float] = { + "you are": 0.1, + "ur a": 0.1, + "such a": 0.2, + "fucking": 0.1, + "damn": 0.2, +} + + +class ContextAnalyzer: + """Analyzes surrounding text to reduce context-dependent false positives.""" + + def __init__(self, config: ContextConfig) -> None: + self.context_window = config.context_window + self.language = config.language + self.domain_whitelists = { + word.lower() for word in (config.domain_whitelists or []) + } + + def analyze_context( + self, text: str, match_word: str, match_index: int + ) -> ContextAnalysisResult: + words = self._tokenize(text) + match_word_index = self._find_word_index(words, match_index) + + if match_word_index == -1: + return ContextAnalysisResult( + context_score=0.5, + reason="Could not locate match in tokenized text", + is_whitelisted=False, + ) + + start_index = max(0, match_word_index - self.context_window) + end_index = min(len(words), match_word_index + self.context_window + 1) + context_words = words[start_index:end_index] + context_text = " ".join(context_words).lower() + + phrase_result = self._check_phrase_context(context_text, match_word) + if phrase_result is not None: + return phrase_result + + if self._is_domain_whitelisted(context_words, match_word): + return ContextAnalysisResult( + context_score=0.8, + reason="Domain-specific whitelist match", + is_whitelisted=True, + ) + + sentiment_score = self._calculate_sentiment_score( + context_words, match_word_index - start_index + ) + return ContextAnalysisResult( + context_score=sentiment_score, + reason=self._generate_reason(sentiment_score, context_words), + is_whitelisted=False, + ) + + def update_domain_whitelist(self, new_whitelist: list[str]) -> None: + self.domain_whitelists = {word.lower() for word in new_whitelist} + + def add_to_domain_whitelist(self, words: list[str]) -> None: + self.domain_whitelists.update(word.lower() for word in words) + + def _check_phrase_context( + self, context_text: str, match_word: str + ) -> ContextAnalysisResult | None: + for phrase, score in POSITIVE_PHRASES.items(): + if match_word in phrase and phrase in context_text: + return ContextAnalysisResult( + context_score=score, + reason=f'Positive phrase detected: "{phrase}"', + is_whitelisted=True, + ) + + for phrase, score in NEGATIVE_PHRASES.items(): + if phrase in context_text: + return ContextAnalysisResult( + context_score=score, + reason=f'Negative phrase detected: "{phrase}"', + is_whitelisted=False, + ) + + return None + + def _is_domain_whitelisted(self, context_words: list[str], match_word: str) -> bool: + normalized_match_word = match_word.lower() + + for word in context_words: + if word in self.domain_whitelists: + return True + if word in GAMING_POSITIVE and normalized_match_word in GAMING_ACCEPTABLE_WORDS: + return True + + return False + + def _generate_reason(self, score: float, context_words: list[str]) -> str: + found_positive = sorted({word for word in context_words if word in POSITIVE_INDICATORS}) + found_negative = sorted({word for word in context_words if word in NEGATIVE_INDICATORS}) + + if score >= 0.7: + details = f" (found: {', '.join(found_positive)})" if found_positive else "" + return f"Positive context detected{details} - likely not profanity" + if score <= 0.3: + details = f" (found: {', '.join(found_negative)})" if found_negative else "" + return f"Negative context detected{details} - likely profanity" + return "Neutral context - uncertain classification" + + def _tokenize(self, text: str) -> list[str]: + normalized = re.sub(r"[^A-Za-z0-9_\s]", " ", text.lower()) + return [word for word in normalized.split() if word] + + def _find_word_index(self, words: list[str], char_index: int) -> int: + current_pos = 0 + for i, word in enumerate(words): + if current_pos >= char_index: + return max(0, i - 1) + current_pos += len(word) + 1 + return len(words) - 1 + + def _calculate_sentiment_score( + self, context_words: list[str], match_position: int + ) -> float: + positive_count = 0.0 + negative_count = 0.0 + total_words = len(context_words) + + for i, word in enumerate(context_words): + distance = abs(i - match_position) + weight = max(0.1, 1 - (distance * 0.2)) + + if word in POSITIVE_INDICATORS: + positive_count += weight + elif word in NEGATIVE_INDICATORS: + negative_count += weight + + total_sentiment = positive_count + negative_count + if total_sentiment == 0: + return 0.5 + + raw_score = positive_count / total_sentiment + adjusted_score = raw_score + + confidence_multiplier = min(1.0, total_words / 5) + adjusted_score = 0.5 + (adjusted_score - 0.5) * confidence_multiplier + + if any(word in {"you", "your", "u", "ur"} for word in context_words) and raw_score < 0.7: + adjusted_score *= 0.7 + + if any( + word in {"movie", "song", "game", "book", "show", "this", "that", "it"} + for word in context_words + ) and raw_score > 0.3: + adjusted_score = min(1.0, adjusted_score * 1.3) + + return max(0.0, min(1.0, adjusted_score)) diff --git a/packages/py/glin_profanity/types/types.py b/packages/py/glin_profanity/types/types.py index 602d5f6..14a78e6 100644 --- a/packages/py/glin_profanity/types/types.py +++ b/packages/py/glin_profanity/types/types.py @@ -106,6 +106,9 @@ class FilterConfig(ContextAwareConfig, total=False): cache_results: bool # Cache profanity check results for repeated strings max_cache_size: int # Maximum cache size when caching is enabled + # Performance options + disable_aho_corasick: bool # Force legacy regex path (testing / debugging) + class FilteredProfanityResult(TypedDict): """Result with minimum severity filtering.""" diff --git a/packages/py/glin_profanity/utils/__init__.py b/packages/py/glin_profanity/utils/__init__.py index 38997c4..d5c9fc3 100644 --- a/packages/py/glin_profanity/utils/__init__.py +++ b/packages/py/glin_profanity/utils/__init__.py @@ -5,6 +5,12 @@ leetspeak detection and Unicode normalization. """ +from .evasion import ( + collapse_separated_characters, + normalize_evasion, + normalize_masked_profanity, + strip_html_and_decode_entities, +) from .leetspeak import ( LeetspeakLevel, collapse_repeated_characters, @@ -12,6 +18,7 @@ contains_leetspeak, generate_leetspeak_variants, normalize_leetspeak, + normalize_leetspeak_variants, ) from .unicode import ( CharacterSetsResult, @@ -23,11 +30,26 @@ normalize_unicode, remove_zero_width_characters, ) +from .word_script import ( + WordScript, + classify_word_script, + has_cjk_word_boundary, + has_latin_word_boundary, + is_cjk_character, + is_latin_word_boundary_before, + match_has_word_boundary, +) __all__ = [ + # Evasion utilities + "normalize_evasion", + "strip_html_and_decode_entities", + "collapse_separated_characters", + "normalize_masked_profanity", # Leetspeak utilities "LeetspeakLevel", "normalize_leetspeak", + "normalize_leetspeak_variants", "collapse_spaced_characters", "collapse_repeated_characters", "contains_leetspeak", @@ -41,4 +63,12 @@ "normalize_nfkd", "contains_unicode_obfuscation", "detect_character_sets", + # Word-script utilities + "WordScript", + "is_cjk_character", + "classify_word_script", + "is_latin_word_boundary_before", + "has_latin_word_boundary", + "has_cjk_word_boundary", + "match_has_word_boundary", ] diff --git a/packages/py/glin_profanity/utils/evasion.py b/packages/py/glin_profanity/utils/evasion.py new file mode 100644 index 0000000..5f72fa5 --- /dev/null +++ b/packages/py/glin_profanity/utils/evasion.py @@ -0,0 +1,60 @@ +"""Evasion normalization for profanity detection.""" + +from __future__ import annotations + +import html +import re + +_SEPARATED_CHAR_PATTERN = re.compile( + r"\b([a-zA-Z0-9@$!#*])(?:[\s._\-]+([a-zA-Z0-9@$!#*])){2,}\b" +) +_NUMERIC_ENTITY_PATTERN = re.compile(r"&#(\d+);") +_HEX_ENTITY_PATTERN = re.compile(r"&#x([0-9a-fA-F]+);") +_MASKED_PATTERNS: list[tuple[re.Pattern[str], str]] = [ + (re.compile(r"\bf\*+cking\b", re.IGNORECASE), "fucking"), + (re.compile(r"\bf\*+ck\b", re.IGNORECASE), "fuck"), + (re.compile(r"\bs\*+hit\b", re.IGNORECASE), "shit"), + (re.compile(r"\bf\*{2,}(?=\W|$)", re.IGNORECASE), "fuck"), + (re.compile(r"\bf\s+yourself\b", re.IGNORECASE), "fuck yourself"), +] + + +def strip_html_and_decode_entities(text: str) -> str: + """Remove HTML tags and decode numeric/named entities.""" + result = re.sub(r"<[^>]*>", "", text) + + def decode_numeric(match: re.Match[str]) -> str: + code = int(match.group(1)) + return chr(code) + + def decode_hex(match: re.Match[str]) -> str: + code = int(match.group(1), 16) + return chr(code) + + result = _NUMERIC_ENTITY_PATTERN.sub(decode_numeric, result) + result = _HEX_ENTITY_PATTERN.sub(decode_hex, result) + return html.unescape(result) + + +def collapse_separated_characters(text: str) -> str: + """Collapse single alphanumerics separated by spaces, dots, underscores, or hyphens.""" + + def replace_match(match: re.Match[str]) -> str: + return re.sub(r"[\s._\-]+", "", match.group(0)) + + return _SEPARATED_CHAR_PATTERN.sub(replace_match, text) + + +def normalize_masked_profanity(text: str) -> str: + """Expand common asterisk-masked profanity abbreviations.""" + result = text + for pattern, replacement in _MASKED_PATTERNS: + result = pattern.sub(replacement, result) + return result + + +def normalize_evasion(text: str) -> str: + """Apply all evasion normalization steps before Unicode/leetspeak handling.""" + result = strip_html_and_decode_entities(text) + result = collapse_separated_characters(result) + return normalize_masked_profanity(result) diff --git a/packages/py/glin_profanity/utils/leetspeak.py b/packages/py/glin_profanity/utils/leetspeak.py index 3d87f5f..d6ba5b1 100644 --- a/packages/py/glin_profanity/utils/leetspeak.py +++ b/packages/py/glin_profanity/utils/leetspeak.py @@ -66,6 +66,61 @@ ] +AGGRESSIVE_VOWEL_SUBSTITUTIONS: dict[str, str] = { + **AGGRESSIVE_SUBSTITUTIONS, + "@": "u", +} + + +def _apply_substitutions(text: str, substitutions: dict[str, str]) -> str: + return "".join(substitutions.get(char, char) for char in text) + + +def _apply_leetspeak_pre_collapse( + text: str, + level: LeetspeakLevel, + remove_spaced_chars: bool, + substitutions: dict[str, str], +) -> str: + normalized = text + + if remove_spaced_chars: + normalized = collapse_spaced_characters(normalized) + + if level == "aggressive": + for pattern, replacement in AGGRESSIVE_MULTI_CHAR: + normalized = re.sub(pattern, replacement, normalized, flags=re.IGNORECASE) + + return _apply_substitutions(normalized, substitutions) + + +def normalize_leetspeak_variants( + text: str, + level: LeetspeakLevel = "moderate", + collapse_repeated: bool = True, + remove_spaced_chars: bool = True, +) -> tuple[str, str]: + """Return normal and aggressive leetspeak variants in one pass.""" + pre_collapsed = _apply_leetspeak_pre_collapse( + text, level, remove_spaced_chars, _get_substitution_map(level) + ) + vowel_pre_collapsed = ( + _apply_leetspeak_pre_collapse( + text, level, remove_spaced_chars, AGGRESSIVE_VOWEL_SUBSTITUTIONS + ) + if level == "aggressive" + else pre_collapsed + ) + + if not collapse_repeated: + return pre_collapsed, vowel_pre_collapsed + + return ( + collapse_repeated_characters(pre_collapsed, 2), + collapse_repeated_characters(vowel_pre_collapsed, 1), + ) + + def normalize_leetspeak( text: str, level: LeetspeakLevel = "moderate", diff --git a/packages/py/glin_profanity/utils/unicode.py b/packages/py/glin_profanity/utils/unicode.py index ae6402a..38ab56e 100644 --- a/packages/py/glin_profanity/utils/unicode.py +++ b/packages/py/glin_profanity/utils/unicode.py @@ -53,6 +53,8 @@ "τ": "t", # Greek small tau "Τ": "T", # Greek capital Tau "υ": "u", # Greek small upsilon + "Ս": "U", # Armenian capital seh + "ս": "u", # Armenian small seh "Υ": "Y", # Greek capital Upsilon "χ": "x", # Greek small chi "Χ": "X", # Greek capital Chi diff --git a/packages/py/glin_profanity/utils/word_script.py b/packages/py/glin_profanity/utils/word_script.py new file mode 100644 index 0000000..3043f21 --- /dev/null +++ b/packages/py/glin_profanity/utils/word_script.py @@ -0,0 +1,76 @@ +"""Word-script classification and boundary checks for Latin vs CJK dictionary entries.""" + +from __future__ import annotations + +import re +from typing import Literal + +WordScript = Literal["latin", "cjk"] + +# Mirrors JavaScript ``\\b`` word character class without the ``u`` flag. +_LATIN_WORD_CHAR = re.compile(r"[A-Za-z0-9_]") + + +def is_cjk_character(char: str) -> bool: + """Return True when the code point belongs to a CJK-related script block.""" + if not char: + return False + + code = ord(char[0]) + return ( + 0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs + or 0x3400 <= code <= 0x4DBF # Extension A + or 0x3040 <= code <= 0x309F # Hiragana + or 0x30A0 <= code <= 0x30FF # Katakana + or 0x31F0 <= code <= 0x31FF # Katakana phonetic extensions + or 0xAC00 <= code <= 0xD7AF # Hangul syllables + or 0x1100 <= code <= 0x11FF # Hangul Jamo + or 0x3130 <= code <= 0x318F # Hangul compatibility Jamo + or 0x3100 <= code <= 0x312F # Bopomofo + or 0xFF66 <= code <= 0xFF9F # Halfwidth katakana + ) + + +def classify_word_script(word: str) -> WordScript: + """ + Classify a dictionary entry by its characters (not by configured language). + + ASCII-only entries from Japanese/Chinese lists still use Latin ``\\b`` rules. + """ + for char in word: + if is_cjk_character(char): + return "cjk" + return "latin" + + +def is_latin_word_boundary_before(text: str, index: int) -> bool: + """JavaScript ``\\b`` boundary at index (between word and non-word ASCII chars).""" + left_is_word = index > 0 and bool(_LATIN_WORD_CHAR.match(text[index - 1])) + right_is_word = index < len(text) and bool(_LATIN_WORD_CHAR.match(text[index])) + return left_is_word != right_is_word + + +def has_latin_word_boundary(text: str, start: int, end: int) -> bool: + return is_latin_word_boundary_before(text, start) and is_latin_word_boundary_before( + text, end + ) + + +def has_cjk_word_boundary(_text: str, _start: int, _end: int) -> bool: + """Substring match anywhere, including ASCII-adjacent or digit-wrapped CJK.""" + return True + + +def match_has_word_boundary( + text: str, + start: int, + end: int, + script: WordScript, + word_boundaries_enabled: bool, +) -> bool: + """Return whether a match at [start, end) satisfies boundary rules for its script.""" + if not word_boundaries_enabled: + return True + if script == "cjk": + return has_cjk_word_boundary(text, start, end) + return has_latin_word_boundary(text, start, end) diff --git a/packages/py/pyproject.toml b/packages/py/pyproject.toml index 459a052..7bdfb8d 100644 --- a/packages/py/pyproject.toml +++ b/packages/py/pyproject.toml @@ -54,6 +54,7 @@ classifiers = [ requires-python = ">=3.10" dependencies = [ "typing-extensions>=4.0.0", + "pyahocorasick>=2.0.0", ] [project.optional-dependencies] diff --git a/packages/py/tests/test_aho_corasick_parity.py b/packages/py/tests/test_aho_corasick_parity.py new file mode 100644 index 0000000..b0b3a91 --- /dev/null +++ b/packages/py/tests/test_aho_corasick_parity.py @@ -0,0 +1,161 @@ +"""Aho-Corasick parity tests between fast and legacy regex paths.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from glin_profanity import Filter +from glin_profanity.types.types import CheckProfanityResult, FilterConfig + +TORTURE_SET_PATH = ( + Path(__file__).resolve().parents[3] / "benchmarks" / "shootout" / "torture-set.json" +) + +FILTER_CONFIGS: list[FilterConfig] = [ + {"languages": ["english"]}, + { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "moderate", + }, + { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + }, + { + "languages": ["english", "spanish"], + "detect_leetspeak": True, + "normalize_unicode": True, + }, + { + "languages": ["english"], + "replace_with": "***", + "detect_leetspeak": True, + "severity_levels": True, + }, + {"all_languages": True, "detect_leetspeak": True, "normalize_unicode": True}, +] + +EXTRA_TEXTS = [ + "", + "hello world", + "The quick brown fox", + "Scunthorpe is a town", + "classic music", + "assassin", + "fuck", + "FUCK", + "f4ck", + "@ss", + "fuuuuck", + "f u c k", + "This is damn bad", +] + + +def _normalize_result(result: CheckProfanityResult) -> dict[str, object]: + return { + "contains_profanity": result["contains_profanity"], + "profane_words": sorted(result.get("profane_words", [])), + "processed_text": result.get("processed_text"), + "severity_map": result.get("severity_map"), + } + + +@pytest.fixture(scope="module") +def torture_set() -> list[dict[str, object]]: + with TORTURE_SET_PATH.open(encoding="utf-8") as handle: + return json.load(handle) + + +@pytest.mark.parametrize("config", FILTER_CONFIGS, ids=lambda cfg: json.dumps(cfg)) +class TestAhoCorasickParity: + def test_uses_aho_corasick_when_eligible(self, config: FilterConfig) -> None: + expects_ac = config.get("word_boundaries", True) or config.get( + "enable_context_aware", False + ) + if expects_ac and not config.get("disable_aho_corasick"): + fast_filter = Filter(config) + assert fast_filter.dictionary_matcher is not None + + @pytest.mark.parametrize("text", EXTRA_TEXTS) + def test_is_profane_parity(self, config: FilterConfig, text: str) -> None: + fast_filter = Filter(config) + legacy_filter = Filter({**config, "disable_aho_corasick": True}) + assert fast_filter.is_profane(text) == legacy_filter.is_profane(text) + + @pytest.mark.parametrize("text", EXTRA_TEXTS) + def test_check_profanity_parity(self, config: FilterConfig, text: str) -> None: + fast_filter = Filter(config) + legacy_filter = Filter({**config, "disable_aho_corasick": True}) + assert _normalize_result(fast_filter.check_profanity(text)) == _normalize_result( + legacy_filter.check_profanity(text) + ) + + def test_torture_set_parity(self, config: FilterConfig, torture_set: list) -> None: + fast_filter = Filter(config) + legacy_filter = Filter({**config, "disable_aho_corasick": True}) + + for case in torture_set: + text = str(case["input"]) + assert fast_filter.is_profane(text) == legacy_filter.is_profane(text) + assert _normalize_result(fast_filter.check_profanity(text)) == _normalize_result( + legacy_filter.check_profanity(text) + ) + + +def test_legacy_path_when_word_boundaries_disabled() -> None: + config: FilterConfig = { + "languages": ["english"], + "word_boundaries": False, + "fuzzy_tolerance_level": 0.6, + } + fast = Filter(config) + legacy = Filter({**config, "disable_aho_corasick": True}) + assert fast.dictionary_matcher is None + assert fast.is_profane("scunthorpe") == legacy.is_profane("scunthorpe") + + +def test_context_aware_mode_uses_aho_corasick_when_word_boundaries_disabled() -> None: + config: FilterConfig = { + "languages": ["english"], + "enable_context_aware": True, + "word_boundaries": False, + "fuzzy_tolerance_level": 0.6, + } + filter_instance = Filter(config) + assert filter_instance.dictionary_matcher is not None + + +def test_context_aware_mode_uses_aho_corasick_for_candidates() -> None: + config: FilterConfig = { + "languages": ["english"], + "enable_context_aware": True, + } + filter_instance = Filter(config) + assert filter_instance.dictionary_matcher is not None + + +def test_context_aware_ac_and_legacy_regex_paths_agree() -> None: + config: FilterConfig = { + "languages": ["english"], + "enable_context_aware": True, + "context_window": 3, + "confidence_threshold": 0.7, + } + fast = Filter(config) + legacy = Filter({**config, "disable_aho_corasick": True}) + cases = [ + "This movie is the bomb", + "The bomb exploded and shit happened", + "You are a fucking idiot", + "This movie is sick!", + ] + + for text in cases: + assert fast.check_profanity(text) == legacy.check_profanity(text) diff --git a/packages/py/tests/test_cjk_matching.py b/packages/py/tests/test_cjk_matching.py new file mode 100644 index 0000000..14e6384 --- /dev/null +++ b/packages/py/tests/test_cjk_matching.py @@ -0,0 +1,128 @@ +"""Tests for CJK automatic matching strategy.""" + +import json + +import pytest + +from glin_profanity import Filter +from glin_profanity.types.types import FilterConfig +from glin_profanity.utils.word_script import ( + classify_word_script, + has_cjk_word_boundary, + has_latin_word_boundary, + is_cjk_character, +) + + +class TestWordScriptUtilities: + def test_is_cjk_character_detects_cjk_scripts(self) -> None: + assert is_cjk_character("你") is True + assert is_cjk_character("エ") is True + assert is_cjk_character("병") is True + assert is_cjk_character("a") is False + + def test_classify_word_script_uses_characters_not_language(self) -> None: + assert classify_word_script("他妈的") == "cjk" + assert classify_word_script("エッチ") == "cjk" + assert classify_word_script("sm") == "latin" + assert classify_word_script("fuck") == "latin" + + def test_has_latin_word_boundary_matches_js_b_semantics(self) -> None: + assert has_latin_word_boundary("hello fuck world", 6, 10) is True + assert has_latin_word_boundary("scunthorpe", 5, 9) is False + assert has_latin_word_boundary("classic", 2, 5) is False + + def test_has_cjk_word_boundary_allows_substring_including_ascii_adjacency( + self, + ) -> None: + assert has_cjk_word_boundary("你他妈的", 1, 4) is True + assert has_cjk_word_boundary("hello操world", 5, 6) is True + assert has_cjk_word_boundary("hello他妈的", 5, 8) is True + assert has_cjk_word_boundary("123エッチ456", 3, 6) is True + + +class TestCjkAutomaticMatchingStrategy: + @classmethod + def setup_class(cls) -> None: + cls.chinese_filter = Filter({"languages": ["chinese"]}) + cls.japanese_filter = Filter({"languages": ["japanese"]}) + cls.korean_filter = Filter({"languages": ["korean"]}) + cls.english_filter = Filter({"languages": ["english"]}) + cls.mixed_filter = Filter({"languages": ["english", "chinese"]}) + + def test_chinese_detection_with_default_word_boundaries(self) -> None: + assert self.chinese_filter.is_profane("你他妈的") is True + assert len(self.chinese_filter.check_profanity("他妈的")["profane_words"]) > 0 + + def test_japanese_detection_with_default_word_boundaries(self) -> None: + assert self.japanese_filter.is_profane("このエッチな話") is True + assert self.japanese_filter.is_profane("エッチ") is True + + def test_korean_detection_with_default_word_boundaries(self) -> None: + assert self.korean_filter.is_profane("이 병신") is True + assert self.korean_filter.is_profane("병신") is True + + def test_japanese_ascii_entries_still_use_latin_boundaries(self) -> None: + assert self.japanese_filter.is_profane("hello xx world") is True + assert self.japanese_filter.is_profane("xxtra") is False + + def test_detects_cjk_profanity_wrapped_in_or_adjacent_to_ascii(self) -> None: + assert self.chinese_filter.is_profane("hello他妈的") is True + assert self.chinese_filter.is_profane("x乳x") is True + assert self.chinese_filter.is_profane("123他妈的456") is True + assert self.japanese_filter.is_profane("abcエッチdef") is True + + def test_does_not_flag_scunthorpe_or_classic_for_english_words(self) -> None: + assert self.english_filter.is_profane("scunthorpe") is False + assert self.english_filter.is_profane("classic") is False + assert self.english_filter.is_profane("assassin") is False + + def test_still_detects_standalone_english_profanity(self) -> None: + assert self.english_filter.is_profane("hello fuck world") is True + + def test_mixed_filter_does_not_false_positive_english_traps(self) -> None: + assert self.mixed_filter.is_profane("scunthorpe") is False + assert self.mixed_filter.is_profane("classic") is False + + def test_detects_both_english_and_chinese_in_one_filter(self) -> None: + assert self.mixed_filter.is_profane("hello fuck") is True + assert self.mixed_filter.is_profane("你他妈的") is True + + def test_replaces_chinese_profanity_without_requiring_b(self) -> None: + filter_instance = Filter( + {"languages": ["chinese"], "replace_with": "***"} + ) + result = filter_instance.check_profanity("你他妈的") + assert result["processed_text"] is not None + assert "***" in result["processed_text"] + assert "他妈的" not in result["processed_text"] + + +class TestCjkAhoCorasickParity: + CASES = [ + "你他妈的", + "他妈的", + "エッチ", + "병신", + "hello fuck", + "scunthorpe", + "hello他妈的", + "x乳x", + "abcエッチdef", + ] + CONFIGS: list[FilterConfig] = [ + {"languages": ["chinese"]}, + {"languages": ["japanese"]}, + {"languages": ["korean"]}, + {"languages": ["english", "chinese"]}, + ] + + @pytest.mark.parametrize("config", CONFIGS, ids=lambda cfg: json.dumps(cfg)) + @pytest.mark.parametrize("text", CASES) + def test_fast_and_legacy_agree(self, config: FilterConfig, text: str) -> None: + fast = Filter(config) + legacy = Filter({**config, "disable_aho_corasick": True}) + assert fast.is_profane(text) == legacy.is_profane(text) + assert fast.check_profanity(text)["contains_profanity"] == legacy.check_profanity( + text + )["contains_profanity"] diff --git a/packages/py/tests/test_context_aware.py b/packages/py/tests/test_context_aware.py new file mode 100644 index 0000000..f4bb747 --- /dev/null +++ b/packages/py/tests/test_context_aware.py @@ -0,0 +1,183 @@ +"""Tests for context-aware filtering.""" + +from glin_profanity import Filter, SeverityLevel + + +class TestContextAwareFiltering: + def setup_method(self) -> None: + self.filter = Filter( + { + "enable_context_aware": True, + "context_window": 3, + "confidence_threshold": 0.7, + "languages": ["english"], + "log_profanity": False, + } + ) + + def test_positive_context_detection(self) -> None: + test_cases = [ + "This movie is sick!", + "This song is the shit!", + "That movie is badass and amazing!", + ] + + for text in test_cases: + result = self.filter.check_profanity(text) + assert result["reason"] is not None + + matches = result.get("matches") + if matches: + for match in matches: + assert match.get("context_score") is not None + assert match.get("reason") is not None + + if result["contains_profanity"] and matches: + has_positive_context = any( + (m.get("context_score") or 0) > 0.3 for m in matches + ) + assert has_positive_context + + def test_gaming_terms_with_positive_context(self) -> None: + gaming_filter = Filter( + { + "enable_context_aware": True, + "context_window": 3, + "domain_whitelists": { + "english": ["player", "gaming", "game"], + }, + "languages": ["english"], + } + ) + + test_cases = [ + "You are a badass player!", + "That was a sick move in the game", + "Your gaming skills are insane!", + ] + + for text in test_cases: + result = gaming_filter.check_profanity(text) + assert result["contains_profanity"] is False + matches = result.get("matches") + if matches: + assert any(m.get("is_whitelisted") for m in matches) + + def test_negative_context_detection(self) -> None: + test_cases = [ + "You are a fucking idiot", + "Such a stupid ass", + "You piece of shit", + ] + + for text in test_cases: + result = self.filter.check_profanity(text) + assert result["contains_profanity"] is True + assert len(result["profane_words"]) > 0 + + context_score = result.get("context_score") + if context_score is not None: + assert context_score < 0.7 + + def test_match_details(self) -> None: + result = self.filter.check_profanity("This is a fucking good movie") + matches = result.get("matches") + assert matches is not None + + if matches: + match = matches[0] + assert match.get("word") is not None + assert match.get("index", -1) >= 0 + assert match.get("severity") is not None + assert match.get("context_score") is not None + assert match.get("reason") is not None + + def test_severity_levels_without_context(self) -> None: + severity_filter = Filter( + { + "enable_context_aware": False, + "severity_levels": True, + "languages": ["english"], + } + ) + + result = severity_filter.check_profanity("This is fucking annoying") + assert result["contains_profanity"] is True + severity_map = result.get("severity_map") + assert severity_map is not None + assert SeverityLevel.EXACT in severity_map.values() + + def test_backward_compatibility_without_context(self) -> None: + traditional_filter = Filter( + {"enable_context_aware": False, "languages": ["english"]} + ) + + result = traditional_filter.check_profanity("This movie is fucking awesome") + assert result["contains_profanity"] is True + assert traditional_filter.is_profane("This movie is fucking awesome") is True + assert result.get("matches") is None + assert result.get("context_score") is None + + def test_is_profane_and_check_profanity_agree_with_context(self) -> None: + result = self.filter.check_profanity("You are a fucking idiot") + assert ( + self.filter.is_profane("You are a fucking idiot") + == result["contains_profanity"] + ) + assert result["contains_profanity"] is True + + clean_result = self.filter.check_profanity("This movie is the bomb") + assert ( + self.filter.is_profane("This movie is the bomb") + == clean_result["contains_profanity"] + ) + assert clean_result["contains_profanity"] is False + + def test_custom_confidence_threshold(self) -> None: + strict_filter = Filter( + { + "enable_context_aware": True, + "confidence_threshold": 0.9, + "languages": ["english"], + } + ) + + result = strict_filter.check_profanity("This movie is fucking awesome") + assert result is not None + assert result["reason"] is not None + + def test_custom_context_window(self) -> None: + narrow_filter = Filter( + { + "enable_context_aware": True, + "context_window": 1, + "languages": ["english"], + } + ) + + result = narrow_filter.check_profanity( + "Amazing! This movie is fucking awesome for sure" + ) + assert result is not None + assert result["reason"] is not None + + def test_empty_string(self) -> None: + result = self.filter.check_profanity("") + assert result["contains_profanity"] is False + assert result["profane_words"] == [] + + def test_single_word(self) -> None: + result = self.filter.check_profanity("shit") + assert result is not None + assert result["reason"] is not None + + def test_very_long_text(self) -> None: + long_text = ( + "This is a very long text that goes on and on about how this movie is " + "absolutely fucking amazing and everyone should watch it because it is " + "amazing and fantastic and wonderful" + ) * 5 + + result = self.filter.check_profanity(long_text) + assert result is not None + assert result["reason"] is not None diff --git a/packages/py/tests/test_context_optimization.py b/packages/py/tests/test_context_optimization.py new file mode 100644 index 0000000..789e7e5 --- /dev/null +++ b/packages/py/tests/test_context_optimization.py @@ -0,0 +1,34 @@ +"""Tests for context-aware optimization (AC candidate discovery + context filter).""" + +from glin_profanity import Filter + + +class TestContextOptimization: + def setup_method(self) -> None: + self.filter = Filter( + { + "enable_context_aware": True, + "languages": ["english"], + } + ) + + def test_uses_aho_corasick_for_candidate_discovery(self) -> None: + assert self.filter.dictionary_matcher is not None + + def test_is_profane_applies_context_filtering(self) -> None: + assert self.filter.is_profane("This movie is the bomb") is False + assert self.filter.is_profane("The bomb exploded and shit happened") is True + assert self.filter.is_profane("You are a fucking idiot") is True + + def test_unrelated_positive_phrase_does_not_whitelist_other_profanity(self) -> None: + text = "The bomb exploded and shit happened" + result = self.filter.check_profanity(text) + + assert result["contains_profanity"] is True + assert "shit" in result["profane_words"] + + def test_relevant_positive_phrase_is_whitelisted(self) -> None: + text = "This movie is the bomb" + result = self.filter.check_profanity(text) + + assert result["contains_profanity"] is False diff --git a/packages/py/tests/test_evasion.py b/packages/py/tests/test_evasion.py new file mode 100644 index 0000000..a61c0a8 --- /dev/null +++ b/packages/py/tests/test_evasion.py @@ -0,0 +1,75 @@ +"""Tests for evasion normalization utilities.""" + +from glin_profanity import Filter +from glin_profanity.utils.evasion import ( + collapse_separated_characters, + normalize_evasion, + strip_html_and_decode_entities, +) +from glin_profanity.utils.unicode import normalize_unicode + + +class TestEvasionUtilities: + def test_strip_html_and_decode_entities(self) -> None: + assert strip_html_and_decode_entities("shit") == "shit" + assert strip_html_and_decode_entities("fuck") == "fuck" + assert strip_html_and_decode_entities("a
ss") == "ass" + + def test_collapse_separated_characters(self) -> None: + assert collapse_separated_characters("f.u.c.k") == "fuck" + assert collapse_separated_characters("f_u_c_k") == "fuck" + assert collapse_separated_characters("b-i-t-c-h") == "bitch" + assert collapse_separated_characters("hello.world") == "hello.world" + + def test_normalize_evasion_pipeline(self) -> None: + assert normalize_evasion("f.u.c.k") == "fuck" + assert normalize_evasion("This is f*cking ridiculous") == "This is fucking ridiculous" + assert normalize_evasion("holy f*** that was amazing") == "holy fuck that was amazing" + assert normalize_evasion("go f yourself") == "go fuck yourself" + + +class TestEvasionFilterDetection: + @classmethod + def setup_class(cls) -> None: + cls.filter = Filter( + { + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, + } + ) + + def test_detects_shootout_miss_categories(self) -> None: + cases = [ + "shi7e", + "fսck", + "f.u.c.k", + "f_u_c_k", + "f-u-c-k", + "s.h.i.t", + "b-i-t-c-h", + "a
ss", + "fuck", + "shit", + "This is f*cking ridiculous", + "holy f*** that was amazing", + "go f yourself", + "what a f@cking mess", + ] + for text in cases: + assert self.filter.is_profane(text), f"expected profane: {text!r}" + + def test_does_not_flag_false_positive_traps(self) -> None: + traps = [ + "scunthorpe", + "classic", + "assassinate", + "Penistone is a town in South Yorkshire", + "shitake mushrooms are delicious", + ] + for text in traps: + assert not self.filter.is_profane(text), f"unexpected profane: {text!r}" + + def test_armenian_homoglyph_normalizes_to_ascii(self) -> None: + assert normalize_unicode("fսck") == "fuck" diff --git a/packages/py/tests/test_filter_pool.py b/packages/py/tests/test_filter_pool.py new file mode 100644 index 0000000..d5c1eef --- /dev/null +++ b/packages/py/tests/test_filter_pool.py @@ -0,0 +1,64 @@ +"""Tests for the shared Filter instance pool.""" + +from glin_profanity import Filter +from glin_profanity.core.filter_pool import ( + FILTER_POOL_MAX, + clear_filter_pool, + create_filter_config, + get_pooled_filter, +) + + +class TestFilterPool: + def setup_method(self) -> None: + clear_filter_pool() + + def teardown_method(self) -> None: + clear_filter_pool() + + def test_returns_same_instance_for_identical_config(self) -> None: + config = {"languages": ["english"]} + first = get_pooled_filter(config) + second = get_pooled_filter(config) + assert first is second + + def test_language_order_does_not_affect_pool_key(self) -> None: + first = get_pooled_filter({"languages": ["english", "spanish"]}) + second = get_pooled_filter({"languages": ["spanish", "english"]}) + assert first is second + + def test_different_configs_return_different_instances(self) -> None: + english = get_pooled_filter({"languages": ["english"]}) + spanish = get_pooled_filter({"languages": ["spanish"]}) + assert english is not spanish + + def test_evicts_oldest_entry_when_pool_is_full(self) -> None: + configs = [{"languages": [f"english"], "custom_words": [f"word{i}"]} for i in range(FILTER_POOL_MAX + 1)] + instances = [get_pooled_filter(config) for config in configs] + oldest = instances[0] + newest = instances[-1] + + assert get_pooled_filter(configs[0]) is not oldest + assert get_pooled_filter(configs[-1]) is newest + + def test_create_filter_config_merges_global_whitelist(self) -> None: + config = create_filter_config({"ignore_words": ["customword"]}) + ignore_words = config.get("ignore_words") or [] + assert "customword" in ignore_words + assert "Class" in ignore_words + + def test_clear_filter_pool(self) -> None: + first = get_pooled_filter({"languages": ["english"]}) + clear_filter_pool() + second = get_pooled_filter({"languages": ["english"]}) + assert first is not second + + def test_pooled_filter_behaves_like_direct_filter(self) -> None: + config = {"languages": ["english"], "enable_context_aware": True} + pooled = get_pooled_filter(config) + direct = Filter(create_filter_config(config)) + text = "You are a fucking idiot" + assert pooled.is_profane(text) == direct.is_profane(text) + assert pooled.check_profanity(text)["contains_profanity"] == direct.check_profanity( + text + )["contains_profanity"] diff --git a/shared/dictionaries/english.json b/shared/dictionaries/english.json index a8aecf6..1ce990e 100644 --- a/shared/dictionaries/english.json +++ b/shared/dictionaries/english.json @@ -329,6 +329,7 @@ "shemale", "shibari", "shit", + "shite", "sh1t", "$hit", "$h!t", diff --git a/tests/cross_language_parity_test.py b/tests/cross_language_parity_test.py index 2308229..ca0e723 100644 --- a/tests/cross_language_parity_test.py +++ b/tests/cross_language_parity_test.py @@ -3,17 +3,105 @@ Ensures Python and JavaScript packages return identical results """ +from __future__ import annotations + import json import subprocess import sys from pathlib import Path -from typing import Dict, Any, List +from typing import Any import pytest -# Add packages to path -sys.path.insert(0, str(Path(__file__).parent.parent / "packages" / "py")) -from glin_profanity import Filter +REPO_ROOT = Path(__file__).resolve().parent.parent +JS_ENTRY = REPO_ROOT / "packages" / "js" / "dist" / "index.cjs" + +sys.path.insert(0, str(REPO_ROOT / "packages" / "py")) +from glin_profanity import Filter # noqa: E402 +from glin_profanity.types.types import SeverityLevel # noqa: E402 + +pytestmark = pytest.mark.skipif( + not JS_ENTRY.exists(), + reason="JavaScript dist not built; run `npm run build` in packages/js", +) + + +def python_config_to_js(config: dict[str, Any]) -> dict[str, Any]: + """Convert snake_case Python config keys to camelCase JS config keys.""" + key_map = { + "replace_with": "replaceWith", + "case_sensitive": "caseSensitive", + "custom_words": "customWords", + "allow_obfuscated_match": "allowObfuscatedMatch", + "word_boundaries": "wordBoundaries", + "fuzzy_tolerance_level": "fuzzyToleranceLevel", + "severity_levels": "severityLevels", + "ignore_words": "ignoreWords", + "all_languages": "allLanguages", + "enable_context_aware": "enableContextAware", + "context_window": "contextWindow", + "confidence_threshold": "confidenceThreshold", + "detect_leetspeak": "detectLeetspeak", + "leetspeak_level": "leetspeakLevel", + "normalize_unicode": "normalizeUnicode", + "cache_results": "cacheResults", + "max_cache_size": "maxCacheSize", + "disable_aho_corasick": "disableAhoCorasick", + "domain_whitelists": "domainWhitelists", + "log_profanity": "logProfanity", + } + + js_config: dict[str, Any] = {} + for key, value in config.items(): + js_config[key_map.get(key, key)] = value + return js_config + + +def run_javascript_node(script: str) -> Any: + result = subprocess.run( + ["node", "-e", script], + capture_output=True, + text=True, + check=True, + cwd=REPO_ROOT, + ) + return json.loads(result.stdout.strip()) + + +def run_javascript_check_profanity(text: str, config: dict[str, Any]) -> dict[str, Any]: + js_config = python_config_to_js(config) + script = f""" +const {{ Filter }} = require({json.dumps(str(JS_ENTRY))}); +const config = {json.dumps(js_config)}; +const filter = new Filter(config); +const result = filter.checkProfanity({json.dumps(text)}); +console.log(JSON.stringify({{ + contains_profanity: result.containsProfanity, + profane_words: result.profaneWords, + processed_text: result.processedText ?? null, + severity_map: result.severityMap ?? {{}}, + reason: result.reason ?? null, + context_score: result.contextScore ?? null, +}})); +""" + return run_javascript_node(script) + + +def run_javascript_is_profane(text: str, config: dict[str, Any]) -> bool: + js_config = python_config_to_js(config) + script = f""" +const {{ Filter }} = require({json.dumps(str(JS_ENTRY))}); +const filter = new Filter({json.dumps(js_config)}); +console.log(filter.isProfane({json.dumps(text)}) ? "true" : "false"); +""" + result = subprocess.run( + ["node", "-e", script], + capture_output=True, + text=True, + check=True, + cwd=REPO_ROOT, + ) + return result.stdout.strip() == "true" class TestCrossLanguageParity: @@ -23,399 +111,339 @@ class TestCrossLanguageParity: { "name": "clean text", "text": "This is a clean message", - "config": {"languages": ["english"]} + "config": {"languages": ["english"]}, }, { - "name": "simple profanity", + "name": "simple profanity", "text": "This is damn bad", - "config": {"languages": ["english"]} + "config": {"languages": ["english"]}, }, { "name": "multiple languages", - "text": "Hello damn world", - "config": {"languages": ["english", "spanish"]} + "text": "Hello damn world", + "config": {"languages": ["english", "spanish"]}, }, { "name": "with replacement", "text": "This is damn annoying", - "config": {"languages": ["english"], "replace_with": "***"} + "config": {"languages": ["english"], "replace_with": "***"}, }, { "name": "case insensitive", "text": "This is DAMN bad", - "config": {"languages": ["english"], "case_sensitive": False} + "config": {"languages": ["english"], "case_sensitive": False}, }, { "name": "custom words", "text": "This contains badword", - "config": {"languages": ["english"], "custom_words": ["badword"]} - } + "config": {"languages": ["english"], "custom_words": ["badword"]}, + }, ] edge_case_tests = [ { "name": "obfuscated profanity - asterisks", "text": "This is d*mn annoying", - "config": {"languages": ["english"], "allow_obfuscated_match": True} + "config": {"languages": ["english"], "allow_obfuscated_match": True}, }, { "name": "obfuscated profanity - numbers", "text": "This is d4mn bad", - "config": {"languages": ["english"], "allow_obfuscated_match": True} + "config": {"languages": ["english"], "allow_obfuscated_match": True}, }, { "name": "obfuscated profanity - symbols", "text": "This is d@mn terrible", - "config": {"languages": ["english"], "allow_obfuscated_match": True} + "config": {"languages": ["english"], "allow_obfuscated_match": True}, }, { "name": "repeated characters", "text": "This is daaaammmn bad", - "config": {"languages": ["english"], "allow_obfuscated_match": True} + "config": {"languages": ["english"], "allow_obfuscated_match": True}, }, { "name": "word boundaries disabled", "text": "This contains helldamn", - "config": {"languages": ["english"], "word_boundaries": False} + "config": {"languages": ["english"], "word_boundaries": False}, }, { "name": "fuzzy matching", "text": "This is dmn bad", - "config": {"languages": ["english"], "fuzzy_tolerance_level": 0.6} + "config": {"languages": ["english"], "fuzzy_tolerance_level": 0.6}, }, { "name": "severity levels enabled", "text": "This is damn bad", - "config": {"languages": ["english"], "severity_levels": True} + "config": {"languages": ["english"], "severity_levels": True}, }, { "name": "ignore words", - "text": "This is damn good", - "config": {"languages": ["english"], "ignore_words": ["damn"]} + "text": "This is damn good", + "config": {"languages": ["english"], "ignore_words": ["damn"]}, }, { "name": "empty text", "text": "", - "config": {"languages": ["english"]} + "config": {"languages": ["english"]}, }, { "name": "whitespace only", "text": " \n\t ", - "config": {"languages": ["english"]} + "config": {"languages": ["english"]}, }, { "name": "mixed case obfuscation", "text": "This is D@MN bad", - "config": {"languages": ["english"], "allow_obfuscated_match": True, "case_sensitive": False} - } + "config": { + "languages": ["english"], + "allow_obfuscated_match": True, + "case_sensitive": False, + }, + }, ] multi_language_tests = [ { "name": "spanish profanity", "text": "Esto es una mierda", - "config": {"languages": ["spanish"]} + "config": {"languages": ["spanish"]}, }, { "name": "french profanity", "text": "C'est de la merde", - "config": {"languages": ["french"]} + "config": {"languages": ["french"]}, }, { "name": "german profanity", "text": "Das ist Scheiße", - "config": {"languages": ["german"]} + "config": {"languages": ["german"]}, }, { "name": "mixed language content", "text": "Hello mierda damn world", - "config": {"languages": ["english", "spanish"]} + "config": {"languages": ["english", "spanish"]}, }, { "name": "all available languages", "text": "This is damn bad", - "config": {"all_languages": True} - } + "config": {"all_languages": True}, + }, ] context_aware_tests = [ { "name": "context aware - enabled", "text": "This is damn good work", - "config": { - "languages": ["english"], + "config": { + "languages": ["english"], "enable_context_aware": True, - "confidence_threshold": 0.7 - } + "confidence_threshold": 0.7, + }, }, { "name": "context aware - disabled", "text": "This is damn good work", - "config": { - "languages": ["english"], - "enable_context_aware": False - } + "config": { + "languages": ["english"], + "enable_context_aware": False, + }, }, { "name": "context window variation", "text": "The damn good weather today", - "config": { - "languages": ["english"], + "config": { + "languages": ["english"], "enable_context_aware": True, "context_window": 5, - "confidence_threshold": 0.8 - } - } + "confidence_threshold": 0.8, + }, + }, + { + "name": "context optimization - whitelisted bomb phrase", + "text": "This movie is the bomb", + "config": { + "languages": ["english"], + "enable_context_aware": True, + }, + }, + { + "name": "context optimization - unrelated positive phrase", + "text": "The bomb exploded and shit happened", + "config": { + "languages": ["english"], + "enable_context_aware": True, + }, + }, + { + "name": "context optimization - negative insult", + "text": "You are a fucking idiot", + "config": { + "languages": ["english"], + "enable_context_aware": True, + }, + }, ] - def run_javascript_test(self, text: str, config: dict) -> dict: - """Run equivalent test in JavaScript and return result.""" - # Convert Python config to JS config (camelCase) - js_config = {} - for key, value in config.items(): - if key == "replace_with": - js_config["replaceWith"] = value - elif key == "case_sensitive": - js_config["caseSensitive"] = value - elif key == "custom_words": - js_config["customWords"] = value - elif key == "allow_obfuscated_match": - js_config["allowObfuscatedMatch"] = value - elif key == "word_boundaries": - js_config["wordBoundaries"] = value - elif key == "fuzzy_tolerance_level": - js_config["fuzzyToleranceLevel"] = value - elif key == "severity_levels": - js_config["severityLevels"] = value - elif key == "ignore_words": - js_config["ignoreWords"] = value - elif key == "all_languages": - js_config["allLanguages"] = value - elif key == "enable_context_aware": - js_config["enableContextAware"] = value - elif key == "context_window": - js_config["contextWindow"] = value - elif key == "confidence_threshold": - js_config["confidenceThreshold"] = value - else: - js_config[key] = value - - js_script = f""" -const {{ Filter }} = require('./packages/js/dist/index.js'); - -const config = {json.dumps(js_config)}; -const filter = new Filter(config); -const result = filter.checkProfanity("{text}"); - -// Convert JS result to match Python format -const output = {{ - "contains_profanity": result.containsProfanity, - "profane_words": result.profaneWords, - "processed_text": result.processedText || null, - "severity_map": result.severityMap || {{}}, - "reason": result.reason || null -}}; -console.log(JSON.stringify(output)); -""" + cjk_matching_tests = [ + { + "name": "chinese substring", + "text": "你他妈的", + "config": {"languages": ["chinese"]}, + }, + { + "name": "chinese ascii adjacency", + "text": "hello他妈的", + "config": {"languages": ["chinese"]}, + }, + { + "name": "chinese wrapped in ascii", + "text": "x乳x", + "config": {"languages": ["chinese"]}, + }, + { + "name": "japanese substring", + "text": "エッチ", + "config": {"languages": ["japanese"]}, + }, + { + "name": "japanese ascii wrapped", + "text": "abcエッチdef", + "config": {"languages": ["japanese"]}, + }, + { + "name": "english scunthorpe trap", + "text": "scunthorpe", + "config": {"languages": ["english"]}, + }, + { + "name": "english standalone profanity", + "text": "hello fuck world", + "config": {"languages": ["english"]}, + }, + { + "name": "mixed english and chinese", + "text": "hello fuck", + "config": {"languages": ["english", "chinese"]}, + }, + ] - try: - result = subprocess.run( - ["node", "-e", js_script], - capture_output=True, - text=True, - check=True + @staticmethod + def assert_check_profanity_parity( + name: str, + py_result: dict[str, Any], + js_result: dict[str, Any], + *, + compare_processed_text: bool = False, + ) -> None: + assert py_result["contains_profanity"] == js_result["contains_profanity"], ( + f"contains_profanity mismatch in {name}" + ) + assert sorted(py_result["profane_words"]) == sorted(js_result["profane_words"]), ( + f"profane_words mismatch in {name}" + ) + assert len(py_result["profane_words"]) == len(js_result["profane_words"]), ( + f"word count mismatch in {name}" + ) + if compare_processed_text: + assert py_result.get("processed_text") == js_result.get("processed_text"), ( + f"processed_text mismatch in {name}" ) - return json.loads(result.stdout.strip()) - except (subprocess.CalledProcessError, json.JSONDecodeError) as e: - pytest.fail(f"JavaScript execution failed: {e}") @pytest.mark.parametrize("test_case", test_cases) - def test_basic_parity_with_javascript(self, test_case): - """Test that Python and JavaScript return identical results for basic cases.""" - name = test_case["name"] - text = test_case["text"] - config = test_case["config"] - - # Python result - py_filter = Filter(config) - py_result = py_filter.check_profanity(text) - - # JavaScript result - js_result = self.run_javascript_test(text, config) - - # Compare core fields that should be identical - assert py_result["contains_profanity"] == js_result["contains_profanity"], \ - f"containsProfanity mismatch in {name}" - - assert sorted(py_result["profane_words"]) == sorted(js_result["profane_words"]), \ - f"profaneWords mismatch in {name}" - - if config.get("replace_with"): - assert py_result.get("processed_text") == js_result.get("processed_text"), \ - f"processedText mismatch in {name}" - - # Both should have same number of detected words - assert len(py_result["profane_words"]) == len(js_result["profane_words"]), \ - f"Word count mismatch in {name}" + def test_basic_parity_with_javascript(self, test_case: dict[str, Any]) -> None: + py_result = Filter(test_case["config"]).check_profanity(test_case["text"]) + js_result = run_javascript_check_profanity(test_case["text"], test_case["config"]) + self.assert_check_profanity_parity( + test_case["name"], + py_result, + js_result, + compare_processed_text=bool(test_case["config"].get("replace_with")), + ) @pytest.mark.parametrize("test_case", edge_case_tests) - def test_edge_case_parity_with_javascript(self, test_case): - """Test that Python and JavaScript handle edge cases identically.""" - name = test_case["name"] - text = test_case["text"] - config = test_case["config"] - - py_filter = Filter(config) - py_result = py_filter.check_profanity(text) - - js_result = self.run_javascript_test(text, config) - - assert py_result["contains_profanity"] == js_result["contains_profanity"], \ - f"containsProfanity mismatch in {name}" - assert sorted(py_result["profane_words"]) == sorted(js_result["profane_words"]), \ - f"profaneWords mismatch in {name}" - - # Test obfuscated detection specifically - if config.get("allow_obfuscated_match") and ('*' in text or '@' in text or '4' in text): - # Should detect obfuscated patterns - assert py_result["contains_profanity"] == js_result["contains_profanity"], \ - f"Obfuscated detection mismatch in {name}" + def test_edge_case_parity_with_javascript(self, test_case: dict[str, Any]) -> None: + py_result = Filter(test_case["config"]).check_profanity(test_case["text"]) + js_result = run_javascript_check_profanity(test_case["text"], test_case["config"]) + self.assert_check_profanity_parity(test_case["name"], py_result, js_result) @pytest.mark.parametrize("test_case", multi_language_tests) - def test_multi_language_parity_with_javascript(self, test_case): - """Test that Python and JavaScript multi-language support is identical.""" - name = test_case["name"] - text = test_case["text"] - config = test_case["config"] - - py_filter = Filter(config) - py_result = py_filter.check_profanity(text) - - js_result = self.run_javascript_test(text, config) - - assert py_result["contains_profanity"] == js_result["contains_profanity"], \ - f"containsProfanity mismatch in {name}" - assert sorted(py_result["profane_words"]) == sorted(js_result["profane_words"]), \ - f"profaneWords mismatch in {name}" - - # Test that both implementations load the same dictionaries - if config.get("all_languages"): - # Should have comprehensive coverage - assert isinstance(py_result["contains_profanity"], bool) - assert isinstance(js_result["contains_profanity"], bool) + def test_multi_language_parity_with_javascript(self, test_case: dict[str, Any]) -> None: + py_result = Filter(test_case["config"]).check_profanity(test_case["text"]) + js_result = run_javascript_check_profanity(test_case["text"], test_case["config"]) + self.assert_check_profanity_parity(test_case["name"], py_result, js_result) @pytest.mark.parametrize("test_case", context_aware_tests) - def test_context_aware_parity_with_javascript(self, test_case): - """Test that Python and JavaScript context analysis is identical.""" - name = test_case["name"] - text = test_case["text"] - config = test_case["config"] - - py_filter = Filter(config) - py_result = py_filter.check_profanity(text) - - js_result = self.run_javascript_test(text, config) - - assert py_result["contains_profanity"] == js_result["contains_profanity"], \ - f"containsProfanity mismatch in {name}" - assert sorted(py_result["profane_words"]) == sorted(js_result["profane_words"]), \ - f"profaneWords mismatch in {name}" - - # Test context-specific behavior - if config.get("enable_context_aware"): - # Both should consider context in filtering decisions - assert "reason" in py_result or py_result.get("reason") is not None - assert "reason" in js_result or js_result.get("reason") is not None - - def test_api_structure_consistency(self): - """Test that Python API has expected structure.""" + def test_context_aware_parity_with_javascript(self, test_case: dict[str, Any]) -> None: + py_result = Filter(test_case["config"]).check_profanity(test_case["text"]) + js_result = run_javascript_check_profanity(test_case["text"], test_case["config"]) + self.assert_check_profanity_parity(test_case["name"], py_result, js_result) + + if test_case["config"].get("enable_context_aware"): + assert py_result.get("reason") is not None + assert js_result.get("reason") is not None + + @pytest.mark.parametrize("test_case", cjk_matching_tests) + def test_cjk_matching_parity_with_javascript(self, test_case: dict[str, Any]) -> None: + py_result = Filter(test_case["config"]).check_profanity(test_case["text"]) + js_result = run_javascript_check_profanity(test_case["text"], test_case["config"]) + self.assert_check_profanity_parity(test_case["name"], py_result, js_result) + + py_is_profane = Filter(test_case["config"]).is_profane(test_case["text"]) + js_is_profane = run_javascript_is_profane(test_case["text"], test_case["config"]) + assert py_is_profane == js_is_profane, f"is_profane mismatch in {test_case['name']}" + + def test_api_structure_consistency(self) -> None: py_filter = Filter({"languages": ["english"]}) py_result = py_filter.check_profanity("test damn") - # Ensure Python result has expected structure assert "contains_profanity" in py_result assert "profane_words" in py_result assert isinstance(py_result["profane_words"], list) - - # Test method names exist assert hasattr(py_filter, "check_profanity") assert hasattr(py_filter, "is_profane") assert hasattr(py_filter, "check_profanity_with_min_severity") - assert callable(py_filter.check_profanity) - assert callable(py_filter.is_profane) - assert callable(py_filter.check_profanity_with_min_severity) - - def test_is_profane_method_parity(self): - """Test that is_profane method returns identical results to JavaScript.""" - test_texts = [ - "clean text", - "damn bad text", - "D@MN obfuscated", - "" + + def test_is_profane_method_parity(self) -> None: + config = {"languages": ["english"], "allow_obfuscated_match": True} + for text in ["clean text", "damn bad text", "D@MN obfuscated", ""]: + py_result = Filter(config).is_profane(text) + js_result = run_javascript_is_profane(text, config) + assert py_result == js_result, f"is_profane mismatch for text: {text!r}" + + def test_is_profane_context_aware_parity(self) -> None: + config = {"languages": ["english"], "enable_context_aware": True} + cases = [ + "This movie is the bomb", + "The bomb exploded and shit happened", + "You are a fucking idiot", ] - - for text in test_texts: - py_filter = Filter({"languages": ["english"], "allow_obfuscated_match": True}) - py_result = py_filter.is_profane(text) - - # Test JavaScript equivalent - js_script = f""" -const {{ Filter }} = require('./packages/js/dist/index.js'); - -const filter = new Filter({{"languages": ["english"], "allowObfuscatedMatch": true}}); -const result = filter.isProfane("{text}"); -console.log(result ? "true" : "false"); -""" - - try: - js_result = subprocess.run( - ["node", "-e", js_script], - capture_output=True, - text=True, - check=True - ) - js_bool = js_result.stdout.strip() == "true" - assert py_result == js_bool, f"is_profane mismatch for text: {text}" - except subprocess.CalledProcessError as e: - pytest.fail(f"JavaScript isProfane test failed: {e}") - - def test_check_profanity_with_min_severity_parity(self): - """Test that check_profanity_with_min_severity returns identical results.""" - from glin_profanity.types.types import SeverityLevel - + for text in cases: + py_result = Filter(config).is_profane(text) + js_result = run_javascript_is_profane(text, config) + assert py_result == js_result, f"context-aware is_profane mismatch for {text!r}" + + def test_check_profanity_with_min_severity_parity(self) -> None: py_filter = Filter({"languages": ["english"], "severity_levels": True}) - py_result = py_filter.check_profanity_with_min_severity("damn bad text", SeverityLevel.EXACT) - - js_script = """ -const { Filter } = require('./packages/js/dist/index.js'); + py_result = py_filter.check_profanity_with_min_severity( + "damn bad text", SeverityLevel.EXACT + ) -const filter = new Filter({"languages": ["english"], "severityLevels": true}); + script = f""" +const {{ Filter }} = require({json.dumps(str(JS_ENTRY))}); +const filter = new Filter({{"languages": ["english"], "severityLevels": true}}); const result = filter.checkProfanityWithMinSeverity("damn bad text", 1); - -const output = { - "filteredWords": result.filteredWords, - "result": { - "containsProfanity": result.result.containsProfanity, - "profaneWords": result.result.profaneWords - } -}; -console.log(JSON.stringify(output)); +console.log(JSON.stringify({{ + filteredWords: result.filteredWords, + result: {{ + containsProfanity: result.result.containsProfanity, + profaneWords: result.result.profaneWords, + }}, +}})); """ - - try: - js_result_raw = subprocess.run( - ["node", "-e", js_script], - capture_output=True, - text=True, - check=True - ) - js_result = json.loads(js_result_raw.stdout.strip()) - - assert sorted(py_result["filtered_words"]) == sorted(js_result["filteredWords"]), \ - "filtered_words mismatch" - assert py_result["result"]["contains_profanity"] == js_result["result"]["containsProfanity"], \ - "result.contains_profanity mismatch" - assert sorted(py_result["result"]["profane_words"]) == sorted(js_result["result"]["profaneWords"]), \ - "result.profane_words mismatch" - except (subprocess.CalledProcessError, json.JSONDecodeError) as e: - pytest.fail(f"JavaScript checkProfanityWithMinSeverity test failed: {e}") \ No newline at end of file + js_result = run_javascript_node(script) + + assert sorted(py_result["filtered_words"]) == sorted(js_result["filteredWords"]) + assert py_result["result"]["contains_profanity"] == js_result["result"]["containsProfanity"] + assert sorted(py_result["result"]["profane_words"]) == sorted( + js_result["result"]["profaneWords"] + ) From a09d389cb040927ee2d9555a6b7af5c5df2cf830 Mon Sep 17 00:00:00 2001 From: wlike Date: Fri, 26 Jun 2026 20:21:37 +0800 Subject: [PATCH 02/11] fix: harden variant mapping, context-aware parity, and filter pool exports Map normalized match spans back to original text for replaceWith and context analysis, supplement AC with fuzzy-only legacy matching when word boundaries are disabled, and align JS/Python filter pool, dedupe, and config export behavior. Co-authored-by: Cursor --- CHANGELOG.md | 15 + packages/js/src/core/filterPool.ts | 2 + packages/js/src/filters/Filter.ts | 410 +++++++++++++----- .../js/src/filters/dictionaryAhoCorasick.ts | 28 +- packages/js/src/nlp/contextAnalyzer.ts | 52 ++- packages/js/src/types/types.ts | 6 + packages/js/src/utils/unicode.ts | 6 +- packages/js/src/utils/variantMapping.ts | 187 ++++++++ packages/js/tests/filter-pool.test.ts | 20 + packages/js/tests/variant-mapping.test.ts | 58 +++ packages/py/glin_profanity/__init__.py | 8 + .../py/glin_profanity/core/filter_pool.py | 1 + .../filters/dictionary_aho_corasick.py | 28 +- packages/py/glin_profanity/filters/filter.py | 368 ++++++++++++---- .../py/glin_profanity/nlp/context_analyzer.py | 50 ++- packages/py/glin_profanity/types/types.py | 3 + packages/py/glin_profanity/utils/unicode.py | 5 + .../glin_profanity/utils/variant_mapping.py | 167 +++++++ packages/py/tests/test_filter_pool_export.py | 22 + packages/py/tests/test_variant_mapping.py | 54 +++ tests/cross_language_parity_test.py | 1 + 21 files changed, 1263 insertions(+), 228 deletions(-) create mode 100644 packages/js/src/utils/variantMapping.ts create mode 100644 packages/js/tests/filter-pool.test.ts create mode 100644 packages/js/tests/variant-mapping.test.ts create mode 100644 packages/py/glin_profanity/utils/variant_mapping.py create mode 100644 packages/py/tests/test_filter_pool_export.py create mode 100644 packages/py/tests/test_variant_mapping.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 480ad92..fd2afbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `ScanMatch.category` now carries pattern family (e.g. `"stripe"`, `"aws_access_key"`) instead of severity - Context-aware mode enables Aho-Corasick candidate discovery even when `wordBoundaries` / `word_boundaries` is `false` - `isProfane` / `is_profane` apply context filtering when `enableContextAware` / `enable_context_aware` is enabled +- Filter instance pools use LRU touch-on-access eviction (JS Map reorder + Python `OrderedDict.move_to_end`) +- Evasion normalization can be disabled via `enableEvasionNormalization` / `enable_evasion_normalization` (default: on) ### Fixed - PI-034 missing `/i` flag (only matched ALL-CAPS variants) @@ -41,6 +43,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Python legacy fuzzy matching restricted to `word_boundaries=false` (aligned with JS) - Latin word-boundary checks use correct start/end positions (fixes Scunthorpe/classic false positives) - Korean NFKD normalization gaps — original/normalized/aggressive three-variant matching in both JS and Python +- Normalized-variant matches map spans back to original text for context analysis and `replaceWith` +- Context-aware recording analyzes context once per match (no duplicate `analyzeContext` calls) +- `checkProfanity` AC path records original matched substrings instead of dictionary keys for normalized variants +- `hasAnyMatch` early-exits on first hit instead of collecting all matches (JS + Python) +- Case-sensitive mode builds Aho-Corasick automaton from original-case dictionary entries +- JS `allowObfuscatedMatch` skipped when `detectLeetspeak` is enabled (parity with Python) +- Python `clear_cache` now clears compiled regex cache +- Python package exports `get_pooled_filter`, `create_filter_config`, and `clear_filter_pool` from top level +- Variant span mapping handles homoglyphs, mask chars, and lowercased checkProfanity tiers +- Context-aware mode supplements AC with legacy fuzzy matching when `wordBoundaries` is disabled +- ContextAnalyzer locates match tokens by character span instead of estimated offsets +- Nested profanity dedupe uses word-boundary checks (avoids `ass` inside `classic`) +- `getConfig()` exports `enableEvasionNormalization` / `enable_evasion_normalization` ## [3.1.0] - 2025-12-30 diff --git a/packages/js/src/core/filterPool.ts b/packages/js/src/core/filterPool.ts index 1ed915a..6effe51 100644 --- a/packages/js/src/core/filterPool.ts +++ b/packages/js/src/core/filterPool.ts @@ -65,6 +65,8 @@ export function getPooledFilter(config: FilterConfig): Filter { const key = configCacheKey(config); const existing = filterPool.get(key); if (existing) { + filterPool.delete(key); + filterPool.set(key, existing); return existing; } diff --git a/packages/js/src/filters/Filter.ts b/packages/js/src/filters/Filter.ts index 4046dc5..9a77b74 100644 --- a/packages/js/src/filters/Filter.ts +++ b/packages/js/src/filters/Filter.ts @@ -4,6 +4,7 @@ import { ContextAnalyzer } from '../nlp/contextAnalyzer'; import { DictionaryAhoCorasick, type DictionaryMatch } from './dictionaryAhoCorasick'; import { normalizeLeetspeak, normalizeLeetspeakVariants } from '../utils/leetspeak'; import { normalizeEvasion } from '../utils/evasion'; +import { mapVariantSpanToOriginal, isNestedProfaneSpan } from '../utils/variantMapping'; import { normalizeUnicode } from '../utils/unicode'; import { classifyWordScript, @@ -51,6 +52,7 @@ class Filter { private detectLeetspeak: boolean; private leetspeakLevel: LeetspeakLevel; private normalizeUnicodeEnabled: boolean; + private enableEvasionNormalization: boolean; // Caching private cacheResults: boolean; private maxCacheSize: number; @@ -118,6 +120,7 @@ class Filter { this.detectLeetspeak = config?.detectLeetspeak ?? false; this.leetspeakLevel = config?.leetspeakLevel ?? 'moderate'; this.normalizeUnicodeEnabled = config?.normalizeUnicode ?? true; + this.enableEvasionNormalization = config?.enableEvasionNormalization ?? true; // Caching settings this.cacheResults = config?.cacheResults ?? false; @@ -148,14 +151,19 @@ class Filter { this.words = new Map(); this.wordScripts = new Map(); + const acWords: string[] = []; + const seenAcKeys = new Set(); for (const word of words) { const key = word.toLowerCase(); this.words.set(key, 1); this.wordScripts.set(key, classifyWordScript(word)); + if (!seenAcKeys.has(key)) { + seenAcKeys.add(key); + acWords.push(this.caseSensitive ? word : key); + } } - this.dictionaryMatcher = this.shouldUseAhoCorasick(config) - ? new DictionaryAhoCorasick(Array.from(this.words.keys())) + ? new DictionaryAhoCorasick(acWords) : null; } @@ -202,7 +210,7 @@ class Filter { * Computes normal and aggressive normalized text in one pass (Unicode + leetspeak). */ private getNormalizedVariants(text: string): { normal: string; aggressive: string } { - let base = normalizeEvasion(text); + let base = this.enableEvasionNormalization ? normalizeEvasion(text) : text; if (this.normalizeUnicodeEnabled) { base = normalizeUnicode(base); @@ -216,7 +224,7 @@ class Filter { }); } - if (this.allowObfuscatedMatch) { + if (this.allowObfuscatedMatch && !this.detectLeetspeak) { const obfuscated = this.normalizeObfuscated(base); return { normal: obfuscated, aggressive: obfuscated }; } @@ -224,6 +232,78 @@ class Filter { return { normal: base, aggressive: base }; } + private forEachTextVariant( + variants: { original: string; normalized: string; aggressive: string }, + callback: (variantText: string, isOriginal: boolean) => boolean | void, + ): boolean { + if (callback(variants.original, true)) { + return true; + } + if (variants.normalized !== variants.original && callback(variants.normalized, false)) { + return true; + } + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original && + callback(variants.aggressive, false) + ) { + return true; + } + return false; + } + + private resolveAcMatchInOriginal( + originalText: string, + variantText: string, + match: DictionaryMatch, + isOriginalVariant: boolean, + ): { matchedWord: string; start: number; end: number } { + if (isOriginalVariant && variantText === originalText) { + return { + matchedWord: match.matchedText, + start: match.start, + end: match.end, + }; + } + + const span = mapVariantSpanToOriginal( + originalText, + variantText, + match.start, + match.end, + ); + return { + matchedWord: span.matchedText, + start: span.start, + end: span.end, + }; + } + + private resolveRegexMatchInOriginal( + originalText: string, + variantText: string, + matchStart: number, + matchEnd: number, + matchedWord: string, + isOriginalVariant: boolean, + ): { matchedWord: string; start: number; end: number } { + if (isOriginalVariant && variantText === originalText) { + return { matchedWord, start: matchStart, end: matchEnd }; + } + + const span = mapVariantSpanToOriginal( + originalText, + variantText, + matchStart, + matchEnd, + ); + return { + matchedWord: span.matchedText, + start: span.start, + end: span.end, + }; + } + /** * Builds the three text variants used for matching (original + normalized + aggressive). */ @@ -277,10 +357,11 @@ class Filter { private collectMatchesFromVariant( dictWord: string, variantText: string, + originalText: string, severity: SeverityLevel, profaneWords: Set, severityMap: Record, - useMatchText: boolean, + isOriginalVariant: boolean, ): void { const regex = this.getRegex(dictWord); const script = this.getWordScript(dictWord); @@ -291,10 +372,17 @@ class Filter { if (!matchHasWordBoundary(variantText, start, end, script, this.wordBoundaries)) { continue; } - const matched = useMatchText ? match[0] : dictWord; - profaneWords.add(matched); - if (severityMap[matched] === undefined) { - severityMap[matched] = severity; + const resolved = this.resolveRegexMatchInOriginal( + originalText, + variantText, + start, + end, + isOriginalVariant ? match[0] : dictWord, + isOriginalVariant, + ); + profaneWords.add(resolved.matchedWord); + if (severityMap[resolved.matchedWord] === undefined) { + severityMap[resolved.matchedWord] = severity; } } } @@ -372,6 +460,7 @@ class Filter { detectLeetspeak: this.detectLeetspeak, leetspeakLevel: this.leetspeakLevel, normalizeUnicode: this.normalizeUnicodeEnabled, + enableEvasionNormalization: this.enableEvasionNormalization, cacheResults: this.cacheResults, maxCacheSize: this.maxCacheSize, }; @@ -524,7 +613,13 @@ class Filter { const variants = this.getTextVariants(value, false); if (this.dictionaryMatcher) { - return this.hasContextAwareAcMatch(value, variants); + if (this.hasContextAwareAcMatch(value, variants)) { + return true; + } + if (!this.wordBoundaries && this.hasContextAwareLegacyFuzzyMatch(value, variants)) { + return true; + } + return false; } return this.hasContextAwareLegacyMatch(value, variants); @@ -537,32 +632,59 @@ class Filter { const options = this.getDictionarySearchOptions(); const matcher = this.dictionaryMatcher!; - const hasFlaggedMatch = ( - variantText: string, - useMatchedText: boolean, - ): boolean => { + return this.forEachTextVariant(variants, (variantText, isOriginal) => { for (const match of matcher.findMatches(variantText, options)) { - const matchedWord = useMatchedText ? match.matchedText : match.dictWord; - if (this.passesContextFilter(text, matchedWord, match.start)) { + const resolved = this.resolveAcMatchInOriginal( + text, + variantText, + match, + isOriginal, + ); + if (this.passesContextFilter(text, resolved.matchedWord, resolved.start)) { return true; } } return false; + }); + } + + private hasContextAwareLegacyFuzzyMatch( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + ): boolean { + for (const dictWord of this.words.keys()) { + if (this.ignoreWords.has(dictWord.toLowerCase())) { + continue; + } + if (this.hasContextAwareLegacyFuzzyMatchForWord(text, variants, dictWord)) { + return true; + } + } + return false; + } + + private hasContextAwareLegacyFuzzyMatchForWord( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + dictWord: string, + ): boolean { + const checkVariant = (variantText: string): boolean => { + if (this.evaluateSeverity(dictWord, variantText) !== SeverityLevel.FUZZY) { + return false; + } + return this.passesContextFilter(text, dictWord, 0); }; - if (hasFlaggedMatch(variants.original, true)) { + if (checkVariant(variants.original)) { return true; } - if ( - variants.normalized !== variants.original && - hasFlaggedMatch(variants.normalized, false) - ) { + if (variants.normalized !== variants.original && checkVariant(variants.normalized)) { return true; } if ( variants.aggressive !== variants.normalized && variants.aggressive !== variants.original && - hasFlaggedMatch(variants.aggressive, false) + checkVariant(variants.aggressive) ) { return true; } @@ -592,12 +714,16 @@ class Filter { variants: { original: string; normalized: string; aggressive: string }, dictWord: string, ): boolean { - const checkVariant = (variantText: string, useMatchText: boolean): boolean => { + const checkVariant = (variantText: string, isOriginal: boolean): boolean => { const severity = this.evaluateSeverity(dictWord, variantText); if (severity === undefined) { return false; } + if (!this.wordBoundaries && severity === SeverityLevel.FUZZY) { + return this.passesContextFilter(text, dictWord, 0); + } + const regex = this.getRegex(dictWord); const script = this.getWordScript(dictWord); let match: RegExpExecArray | null; @@ -607,8 +733,15 @@ class Filter { if (!matchHasWordBoundary(variantText, start, end, script, this.wordBoundaries)) { continue; } - const matchedWord = useMatchText ? match[0] : dictWord; - if (this.passesContextFilter(text, matchedWord, start)) { + const resolved = this.resolveRegexMatchInOriginal( + text, + variantText, + start, + end, + isOriginal ? match[0] : dictWord, + isOriginal, + ); + if (this.passesContextFilter(text, resolved.matchedWord, resolved.start)) { return true; } } @@ -634,24 +767,11 @@ class Filter { private isProfaneWithAhoCorasick(value: string): boolean { const variants = this.getTextVariants(value, false); const options = this.getDictionarySearchOptions(); + const matcher = this.dictionaryMatcher!; - if (this.dictionaryMatcher!.hasAnyMatch(variants.original, options)) { - return true; - } - if ( - variants.normalized !== variants.original && - this.dictionaryMatcher!.hasAnyMatch(variants.normalized, options) - ) { - return true; - } - if ( - variants.aggressive !== variants.normalized && - variants.aggressive !== variants.original && - this.dictionaryMatcher!.hasAnyMatch(variants.aggressive, options) - ) { - return true; - } - return false; + return this.forEachTextVariant(variants, (variantText) => + matcher.hasAnyMatch(variantText, options), + ); } private isProfaneLegacy(value: string): boolean { @@ -677,34 +797,26 @@ class Filter { const profaneWords = new Set(); const severityMap: Record = {}; const options = this.getDictionarySearchOptions(); + const matcher = this.dictionaryMatcher!; - for (const match of this.dictionaryMatcher!.findMatches(variants.original, options)) { - profaneWords.add(match.matchedText); - if (severityMap[match.matchedText] === undefined) { - severityMap[match.matchedText] = SeverityLevel.EXACT; - } - } - - if (variants.normalized !== variants.original) { - for (const match of this.dictionaryMatcher!.findMatches(variants.normalized, options)) { - profaneWords.add(match.dictWord); - if (severityMap[match.dictWord] === undefined) { - severityMap[match.dictWord] = SeverityLevel.EXACT; + this.forEachTextVariant(variants, (variantText, isOriginal) => { + for (const match of matcher.findMatches(variantText, options)) { + const resolved = this.resolveAcMatchInOriginal( + text, + variantText, + match, + isOriginal, + ); + if (!resolved.matchedWord) { + continue; } - } - } - - if ( - variants.aggressive !== variants.normalized && - variants.aggressive !== variants.original - ) { - for (const match of this.dictionaryMatcher!.findMatches(variants.aggressive, options)) { - profaneWords.add(match.dictWord); - if (severityMap[match.dictWord] === undefined) { - severityMap[match.dictWord] = SeverityLevel.EXACT; + profaneWords.add(resolved.matchedWord); + if (severityMap[resolved.matchedWord] === undefined) { + severityMap[resolved.matchedWord] = SeverityLevel.EXACT; } } - } + return false; + }); return this.buildProfanityResult(text, profaneWords, severityMap); } @@ -722,6 +834,7 @@ class Filter { this.collectMatchesFromVariant( dictWord, variants.original, + text, severity, profaneWords, severityMap, @@ -735,6 +848,7 @@ class Filter { this.collectMatchesFromVariant( dictWord, variants.normalized, + text, severity, profaneWords, severityMap, @@ -749,10 +863,15 @@ class Filter { ) { severity = this.evaluateSeverity(dictWord, variants.aggressive); if (severity !== undefined) { - profaneWords.add(dictWord); - if (severityMap[dictWord] === undefined) { - severityMap[dictWord] = SeverityLevel.EXACT; - } + this.collectMatchesFromVariant( + dictWord, + variants.aggressive, + text, + severity, + profaneWords, + severityMap, + false, + ); } } } @@ -790,7 +909,10 @@ class Filter { matchObj.contextScore = contextResult.contextScore; matchObj.reason = contextResult.reason; matchObj.isWhitelisted = contextResult.isWhitelisted; - if (!this.passesContextFilter(text, matchedWord, matchIndex)) { + if ( + contextResult.isWhitelisted || + contextResult.contextScore > this.confidenceThreshold + ) { return; } } @@ -814,35 +936,69 @@ class Filter { const options = this.getDictionarySearchOptions(); const matcher = this.dictionaryMatcher!; - const processAcMatch = (match: DictionaryMatch, useMatchedText: boolean) => { - this.recordContextAwareMatch( - text, - useMatchedText ? match.matchedText : match.dictWord, - match.start, - SeverityLevel.EXACT, - profaneWords, - severityMap, - matches, - seen, - ); - }; + this.forEachTextVariant(variants, (variantText, isOriginal) => { + for (const match of matcher.findMatches(variantText, options)) { + const resolved = this.resolveAcMatchInOriginal( + text, + variantText, + match, + isOriginal, + ); + this.recordContextAwareMatch( + text, + resolved.matchedWord, + resolved.start, + SeverityLevel.EXACT, + profaneWords, + severityMap, + matches, + seen, + ); + } + return false; + }); + } - for (const match of matcher.findMatches(variants.original, options)) { - processAcMatch(match, true); - } + private collectContextAwareCandidatesFromLegacyFuzzy( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + profaneWords: string[], + severityMap: Record, + matches: Match[], + seen: Set, + ): void { + for (const dictWord of this.words.keys()) { + if (this.ignoreWords.has(dictWord.toLowerCase())) { + continue; + } - if (variants.normalized !== variants.original) { - for (const match of matcher.findMatches(variants.normalized, options)) { - processAcMatch(match, false); + const collectFromVariant = (variantText: string) => { + if (this.evaluateSeverity(dictWord, variantText) !== SeverityLevel.FUZZY) { + return; + } + this.recordContextAwareMatch( + text, + dictWord, + 0, + SeverityLevel.FUZZY, + profaneWords, + severityMap, + matches, + seen, + ); + }; + + collectFromVariant(variants.original); + + if (variants.normalized !== variants.original) { + collectFromVariant(variants.normalized); } - } - if ( - variants.aggressive !== variants.normalized && - variants.aggressive !== variants.original - ) { - for (const match of matcher.findMatches(variants.aggressive, options)) { - processAcMatch(match, false); + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original + ) { + collectFromVariant(variants.aggressive); } } } @@ -862,13 +1018,27 @@ class Filter { const collectFromVariant = ( variantText: string, - useMatchText: boolean, + isOriginal: boolean, ) => { const severity = this.evaluateSeverity(dictWord, variantText); if (severity === undefined) { return; } + if (!this.wordBoundaries && severity === SeverityLevel.FUZZY) { + this.recordContextAwareMatch( + text, + dictWord, + 0, + severity, + profaneWords, + severityMap, + matches, + seen, + ); + return; + } + const regex = this.getRegex(dictWord); const script = this.getWordScript(dictWord); let match: RegExpExecArray | null; @@ -878,10 +1048,18 @@ class Filter { if (!matchHasWordBoundary(variantText, start, end, script, this.wordBoundaries)) { continue; } - this.recordContextAwareMatch( + const resolved = this.resolveRegexMatchInOriginal( text, - useMatchText ? match[0] : dictWord, + variantText, start, + end, + isOriginal ? match[0] : dictWord, + isOriginal, + ); + this.recordContextAwareMatch( + text, + resolved.matchedWord, + resolved.start, severity, profaneWords, severityMap, @@ -915,7 +1093,9 @@ class Filter { let processedText = text; if (this.replaceWith && profaneWords.length > 0) { - const uniqueWords = Array.from(new Set(profaneWords)); + const uniqueWords = this.dedupeNestedProfaneWords( + Array.from(new Set(profaneWords)), + ).sort((a, b) => b.length - a.length); for (const word of uniqueWords) { processedText = processedText.replace( this.getReplacementRegex(word), @@ -935,13 +1115,16 @@ class Filter { return { containsProfanity: profaneWords.length > 0, - profaneWords: Array.from(new Set(profaneWords)), + profaneWords: this.dedupeNestedProfaneWords(Array.from(new Set(profaneWords))).sort(), processedText: this.replaceWith ? processedText : undefined, severityMap: this.severityLevels && Object.keys(severityMap).length > 0 ? severityMap : undefined, - matches: matches.length > 0 ? matches : undefined, + matches: + matches.length > 0 + ? [...matches].sort((a, b) => a.index - b.index || a.word.localeCompare(b.word)) + : undefined, contextScore, reason: matches.length > 0 @@ -966,6 +1149,16 @@ class Filter { matches, seen, ); + if (!this.wordBoundaries) { + this.collectContextAwareCandidatesFromLegacyFuzzy( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + ); + } } else { this.collectContextAwareCandidatesFromLegacy( text, @@ -996,12 +1189,23 @@ class Filter { return new RegExp(`\\b${escaped}\\b`, 'gi'); } + private dedupeNestedProfaneWords(words: string[]): string[] { + return words.filter( + (word) => + !words.some( + (other) => other !== word && isNestedProfaneSpan(word, other), + ), + ); + } + private buildProfanityResult( text: string, profaneWords: Set, severityMap: Record, ): CheckProfanityResult { - const profaneWordList = Array.from(profaneWords); + const profaneWordList = this.dedupeNestedProfaneWords(Array.from(profaneWords)).sort( + (a, b) => b.length - a.length, + ); let processedText = text; if (this.replaceWith && profaneWordList.length > 0) { diff --git a/packages/js/src/filters/dictionaryAhoCorasick.ts b/packages/js/src/filters/dictionaryAhoCorasick.ts index 7292e4c..d0b60c6 100644 --- a/packages/js/src/filters/dictionaryAhoCorasick.ts +++ b/packages/js/src/filters/dictionaryAhoCorasick.ts @@ -1,5 +1,6 @@ import AhoCorasick from 'modern-ahocorasick'; import { + classifyWordScript, matchHasWordBoundary, type WordScript, } from '../utils/wordScript'; @@ -81,11 +82,34 @@ export class DictionaryAhoCorasick { } hasAnyMatch(text: string, options: DictionarySearchOptions): boolean { - return this.findMatches(text, options).length > 0; + const haystack = options.caseSensitive ? text : text.toLowerCase(); + + for (const [endGraphemeIdx, dictWords] of this.ac.search(haystack)) { + for (const dictWord of dictWords) { + if (options.ignoreWords.has(dictWord.toLowerCase())) { + continue; + } + + const wordLen = this.wordGraphemeLengths.get(dictWord) ?? countGraphemes(dictWord); + const startGrapheme = endGraphemeIdx - wordLen + 1; + const start = graphemeStartToStringIndex(text, startGrapheme); + const end = graphemeEndToExclusiveStringIndex(text, endGraphemeIdx); + const script = + options.wordScripts.get(dictWord.toLowerCase()) ?? + classifyWordScript(dictWord); + + if (matchHasWordBoundary(text, start, end, script, options.wordBoundaries)) { + return true; + } + } + } + + return false; } findMatches(text: string, options: DictionarySearchOptions): DictionaryMatch[] { const haystack = options.caseSensitive ? text : text.toLowerCase(); + // modern-ahocorasick segments by grapheme and returns grapheme indices const hits = this.ac.search(haystack); const results: DictionaryMatch[] = []; const seen = new Set(); @@ -96,7 +120,7 @@ export class DictionaryAhoCorasick { continue; } - const wordLen = this.wordGraphemeLengths.get(dictWord) ?? dictWord.length; + const wordLen = this.wordGraphemeLengths.get(dictWord) ?? countGraphemes(dictWord); const startGrapheme = endGraphemeIdx - wordLen + 1; const start = graphemeStartToStringIndex(text, startGrapheme); const end = graphemeEndToExclusiveStringIndex(text, endGraphemeIdx); diff --git a/packages/js/src/nlp/contextAnalyzer.ts b/packages/js/src/nlp/contextAnalyzer.ts index 15158d6..1c7d9fa 100644 --- a/packages/js/src/nlp/contextAnalyzer.ts +++ b/packages/js/src/nlp/contextAnalyzer.ts @@ -92,8 +92,8 @@ export class ContextAnalyzer { matchWord: string, matchIndex: number ): ContextAnalysisResult { - const words = this.tokenize(text); - const matchWordIndex = this.findWordIndex(words, matchIndex); + const tokens = this.tokenizeWithSpans(text); + const matchWordIndex = this.findWordIndex(tokens, matchIndex); if (matchWordIndex === -1) { return { @@ -105,8 +105,8 @@ export class ContextAnalyzer { // Extract context window const startIndex = Math.max(0, matchWordIndex - this.contextWindow); - const endIndex = Math.min(words.length, matchWordIndex + this.contextWindow + 1); - const contextWords = words.slice(startIndex, endIndex); + const endIndex = Math.min(tokens.length, matchWordIndex + this.contextWindow + 1); + const contextWords = tokens.slice(startIndex, endIndex).map((token) => token.word); const contextText = contextWords.join(' ').toLowerCase(); // Check for exact phrase matches first @@ -195,28 +195,42 @@ export class ContextAnalyzer { } } - private tokenize(text: string): string[] { - // Simple tokenization - split on whitespace and punctuation - return text.toLowerCase() - .replace(/[^\w\s]/g, ' ') - .split(/\s+/) - .filter(word => word.length > 0); + private tokenizeWithSpans(text: string): Array<{ word: string; start: number; end: number }> { + const tokens: Array<{ word: string; start: number; end: number }> = []; + const pattern = /\S+/g; + let match: RegExpExecArray | null; + + while ((match = pattern.exec(text)) !== null) { + const raw = match[0]; + const normalized = raw.toLowerCase().replace(/[^\p{L}\p{N}_]/gu, ''); + if (normalized.length > 0) { + tokens.push({ + word: normalized, + start: match.index, + end: match.index + raw.length, + }); + } + } + + return tokens; } - private findWordIndex(words: string[], charIndex: number): number { - // This is a simplified approach - in production, you'd want more robust mapping - // For now, we'll estimate based on the character position - let currentPos = 0; - for (let i = 0; i < words.length; i++) { - if (currentPos >= charIndex) { + private findWordIndex( + tokens: Array<{ word: string; start: number; end: number }>, + charIndex: number, + ): number { + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]!; + if (charIndex >= token.start && charIndex < token.end) { + return i; + } + if (charIndex < token.start) { return Math.max(0, i - 1); } - currentPos += words[i].length + 1; // +1 for space } - return words.length - 1; + return Math.max(0, tokens.length - 1); } - private calculateSentimentScore(contextWords: string[], matchPosition: number): number { let positiveCount = 0; let negativeCount = 0; diff --git a/packages/js/src/types/types.ts b/packages/js/src/types/types.ts index d1844d0..c255709 100644 --- a/packages/js/src/types/types.ts +++ b/packages/js/src/types/types.ts @@ -121,6 +121,12 @@ export interface FilterConfig extends ContextAwareConfig { * @internal */ disableAhoCorasick?: boolean; + + /** + * Apply evasion normalization (HTML/separators/masking) before matching. + * @default true + */ + enableEvasionNormalization?: boolean; } /** Result with minimum severity filtering */ diff --git a/packages/js/src/utils/unicode.ts b/packages/js/src/utils/unicode.ts index f3c76af..7b4c6fb 100644 --- a/packages/js/src/utils/unicode.ts +++ b/packages/js/src/utils/unicode.ts @@ -277,10 +277,14 @@ export function convertFullWidth(text: string): string { * @param text - The input text * @returns Text with homoglyphs converted */ +export function homoglyphToAscii(char: string): string { + return HOMOGLYPHS[char] ?? char; +} + export function convertHomoglyphs(text: string): string { return text .split('') - .map((char) => HOMOGLYPHS[char] || char) + .map((char) => homoglyphToAscii(char)) .join(''); } diff --git a/packages/js/src/utils/variantMapping.ts b/packages/js/src/utils/variantMapping.ts new file mode 100644 index 0000000..7a0d7d8 --- /dev/null +++ b/packages/js/src/utils/variantMapping.ts @@ -0,0 +1,187 @@ +/** + * Maps span positions from a normalized variant back to the original text. + * Assumes normalization is order-preserving (no reordering of characters). + */ + +import { homoglyphToAscii } from './unicode'; + +export interface OriginalSpan { + start: number; + end: number; + matchedText: string; +} + +const LEET_TO_ASCII: Record = { + '@': 'a', + $: 's', + '!': 'i', + '1': 'i', + '0': 'o', + '3': 'e', + '4': 'a', + '5': 's', + '7': 't', + '8': 'b', + '9': 'g', +}; + +const SKIPPABLE_ORIGINAL_CHARS = new Set(['*', '.', '_', '-', ' ']); + +function charsEqual(a: string, b: string): boolean { + return a === b || a.toLowerCase() === b.toLowerCase(); +} + +function charsAlign(originalChar: string, variantChar: string): boolean { + if (charsEqual(originalChar, variantChar)) { + return true; + } + + const leet = LEET_TO_ASCII[originalChar] ?? LEET_TO_ASCII[originalChar.toLowerCase()]; + if (leet !== undefined && charsEqual(leet, variantChar)) { + return true; + } + + const homoglyph = homoglyphToAscii(originalChar); + if (homoglyph !== originalChar && charsEqual(homoglyph, variantChar)) { + return true; + } + + return false; +} + +function isSkippableOriginalChar(char: string): boolean { + return SKIPPABLE_ORIGINAL_CHARS.has(char); +} + +function fallbackSpan( + original: string, + variant: string, + variantStart: number, + variantEnd: number, +): OriginalSpan { + const needle = variant.slice(variantStart, variantEnd); + if (!needle) { + return { start: 0, end: 0, matchedText: '' }; + } + + const lowerOriginal = original.toLowerCase(); + const lowerNeedle = needle.toLowerCase(); + const idx = lowerOriginal.indexOf(lowerNeedle); + if (idx >= 0) { + return { + start: idx, + end: idx + needle.length, + matchedText: original.slice(idx, idx + needle.length), + }; + } + + return { + start: Math.min(variantStart, original.length), + end: Math.min(variantEnd, original.length), + matchedText: original.slice( + Math.min(variantStart, original.length), + Math.min(variantEnd, original.length), + ), + }; +} + +/** + * Walk original and variant in parallel to locate the original span + * corresponding to [variantStart, variantEnd) in the variant string. + */ +export function mapVariantSpanToOriginal( + original: string, + variant: string, + variantStart: number, + variantEnd: number, +): OriginalSpan { + if (variantStart >= variantEnd) { + return { start: 0, end: 0, matchedText: '' }; + } + + if (original === variant) { + return { + start: variantStart, + end: variantEnd, + matchedText: original.slice(variantStart, variantEnd), + }; + } + + let originalIndex = 0; + let variantIndex = 0; + let origStart = -1; + let origEnd = -1; + + while (variantIndex < variant.length && originalIndex <= original.length) { + if (variantIndex === variantStart && origStart === -1) { + origStart = originalIndex; + } + if (variantIndex === variantEnd) { + origEnd = originalIndex; + break; + } + + if (originalIndex >= original.length) { + variantIndex++; + continue; + } + + const variantChar = variant[variantIndex]!; + const originalChar = original[originalIndex]!; + + if (charsAlign(originalChar, variantChar)) { + variantIndex++; + originalIndex++; + continue; + } + + if (isSkippableOriginalChar(originalChar)) { + originalIndex++; + continue; + } + + originalIndex++; + } + + if (origStart === -1) { + origStart = 0; + } + if (origEnd === -1) { + origEnd = original.length; + } + + const matchedText = original.slice(origStart, origEnd); + if (!matchedText || origStart >= origEnd) { + return fallbackSpan(original, variant, variantStart, variantEnd); + } + + return { start: origStart, end: origEnd, matchedText }; +} + +/** True when `shorter` is a word-bounded substring of `longer` (avoids ass/classic). */ +export function isNestedProfaneSpan(shorter: string, longer: string): boolean { + if (longer.length <= shorter.length) { + return false; + } + + let searchFrom = 0; + while (searchFrom <= longer.length - shorter.length) { + const idx = longer.indexOf(shorter, searchFrom); + if (idx === -1) { + return false; + } + + const beforeOk = idx === 0 || !/\w/.test(longer[idx - 1]!); + const afterIdx = idx + shorter.length; + const afterOk = + afterIdx === longer.length || !/\w/.test(longer[afterIdx]!); + + if (beforeOk && afterOk) { + return true; + } + + searchFrom = idx + 1; + } + + return false; +} diff --git a/packages/js/tests/filter-pool.test.ts b/packages/js/tests/filter-pool.test.ts new file mode 100644 index 0000000..cd62bc6 --- /dev/null +++ b/packages/js/tests/filter-pool.test.ts @@ -0,0 +1,20 @@ +import { clearFilterPool, getPooledFilter } from '../src/core/filterPool'; + +describe('filterPool', () => { + afterEach(() => { + clearFilterPool(); + }); + + test('reuses the same Filter instance for identical config', () => { + const config = { languages: ['english'] as ('english')[] }; + const first = getPooledFilter(config); + const second = getPooledFilter(config); + expect(second).toBe(first); + }); + + test('creates separate instances for different configs', () => { + const english = getPooledFilter({ languages: ['english'] }); + const spanish = getPooledFilter({ languages: ['spanish'] }); + expect(spanish).not.toBe(english); + }); +}); diff --git a/packages/js/tests/variant-mapping.test.ts b/packages/js/tests/variant-mapping.test.ts new file mode 100644 index 0000000..504713d --- /dev/null +++ b/packages/js/tests/variant-mapping.test.ts @@ -0,0 +1,58 @@ +import { mapVariantSpanToOriginal, isNestedProfaneSpan } from '../src/utils/variantMapping'; + +describe('mapVariantSpanToOriginal', () => { + test('maps collapsed separators back to original span', () => { + const original = 'say f.u.c.k off'; + const variant = 'say fuck off'; + const span = mapVariantSpanToOriginal(original, variant, 4, 8); + expect(span.matchedText).toBe('f.u.c.k'); + expect(original.slice(span.start, span.end)).toBe('f.u.c.k'); + }); + + test('maps leetspeak substitutions back to original span', () => { + const span = mapVariantSpanToOriginal('@ss', 'ass', 0, 3); + expect(span.matchedText).toBe('@ss'); + }); + + test('maps repeated-character collapse back to original span', () => { + const span = mapVariantSpanToOriginal('fuuuuuck', 'fuck', 0, 4); + expect(span.matchedText).toBe('fuuuuuck'); + }); + + test('maps asterisk masking back to original span', () => { + const span = mapVariantSpanToOriginal('holy f***', 'holy fuck', 5, 9); + expect(span.matchedText).toBe('f***'); + }); + + test('maps unicode homoglyphs back to original span', () => { + const original = 'fυck you'; + const variant = 'fuck you'; + const span = mapVariantSpanToOriginal(original, variant, 0, 4); + expect(span.matchedText).toBe('fυck'); + }); + + test('maps lowercased original tier back to mixed-case text', () => { + const original = 'What a FUCK'; + const variant = 'what a fuck'; + const span = mapVariantSpanToOriginal(original, variant, 7, 11); + expect(span.matchedText).toBe('FUCK'); + }); + + test('returns identity mapping when texts match', () => { + const text = 'plain fuck here'; + const span = mapVariantSpanToOriginal(text, text, 6, 10); + expect(span.matchedText).toBe('fuck'); + expect(span.start).toBe(6); + expect(span.end).toBe(10); + }); +}); + +describe('isNestedProfaneSpan', () => { + test('detects nested profanity spans with word boundaries', () => { + expect(isNestedProfaneSpan('sh!t', 'piece of sh!t')).toBe(true); + }); + + test('does not treat ass in classic as nested', () => { + expect(isNestedProfaneSpan('ass', 'classic')).toBe(false); + }); +}); diff --git a/packages/py/glin_profanity/__init__.py b/packages/py/glin_profanity/__init__.py index a6981c6..561f2a5 100644 --- a/packages/py/glin_profanity/__init__.py +++ b/packages/py/glin_profanity/__init__.py @@ -19,6 +19,11 @@ __email__ = "contact@glincker.com" from .filters.filter import Filter +from .core.filter_pool import ( + clear_filter_pool, + create_filter_config, + get_pooled_filter, +) from .types.types import ( CheckProfanityResult, FilterConfig, @@ -47,6 +52,9 @@ __all__ = [ # Core "Filter", + "create_filter_config", + "get_pooled_filter", + "clear_filter_pool", # Types "CheckProfanityResult", "FilterConfig", diff --git a/packages/py/glin_profanity/core/filter_pool.py b/packages/py/glin_profanity/core/filter_pool.py index 6af1318..16bd9cd 100644 --- a/packages/py/glin_profanity/core/filter_pool.py +++ b/packages/py/glin_profanity/core/filter_pool.py @@ -80,6 +80,7 @@ def get_pooled_filter(config: FilterConfig | None = None) -> Filter: existing = _filter_pool.get(key) if existing is not None: + _filter_pool.move_to_end(key) return existing filter_instance = Filter(effective) diff --git a/packages/py/glin_profanity/filters/dictionary_aho_corasick.py b/packages/py/glin_profanity/filters/dictionary_aho_corasick.py index cfd0166..5173dfc 100644 --- a/packages/py/glin_profanity/filters/dictionary_aho_corasick.py +++ b/packages/py/glin_profanity/filters/dictionary_aho_corasick.py @@ -103,7 +103,29 @@ def __init__(self, words: list[str]) -> None: } def has_any_match(self, text: str, options: DictionarySearchOptions) -> bool: - return bool(self.find_matches(text, options)) + haystack = text if options.case_sensitive else text.lower() + + for end_index_inclusive, dict_word in self._automaton.iter(haystack): + if dict_word.lower() in options.ignore_words: + continue + + word_len = self._word_grapheme_lengths.get( + dict_word, _count_graphemes(dict_word) + ) + end_grapheme = _code_point_end_to_grapheme_end(haystack, end_index_inclusive) + start_grapheme = end_grapheme - word_len + 1 + start = _grapheme_start_to_string_index(text, start_grapheme) + end = _grapheme_end_to_exclusive_string_index(text, end_grapheme) + script = options.word_scripts.get(dict_word.lower()) or classify_word_script( + dict_word + ) + + if match_has_word_boundary( + text, start, end, script, options.word_boundaries + ): + return True + + return False def find_matches( self, text: str, options: DictionarySearchOptions @@ -121,7 +143,9 @@ def find_matches( start_grapheme = end_grapheme - word_len + 1 start = _grapheme_start_to_string_index(text, start_grapheme) end = _grapheme_end_to_exclusive_string_index(text, end_grapheme) - script = options.word_scripts.get(dict_word, "latin") + script = options.word_scripts.get(dict_word.lower()) or classify_word_script( + dict_word + ) if not match_has_word_boundary( text, start, end, script, options.word_boundaries diff --git a/packages/py/glin_profanity/filters/filter.py b/packages/py/glin_profanity/filters/filter.py index 5bd6902..6b7b4b2 100644 --- a/packages/py/glin_profanity/filters/filter.py +++ b/packages/py/glin_profanity/filters/filter.py @@ -3,7 +3,7 @@ from __future__ import annotations import re -from typing import Literal +from typing import Callable, Literal from glin_profanity.data.dictionary import dictionary from glin_profanity.filters.dictionary_aho_corasick import ( @@ -20,6 +20,10 @@ SeverityLevel, ) from glin_profanity.utils.evasion import normalize_evasion +from glin_profanity.utils.variant_mapping import ( + is_nested_profane_span, + map_variant_span_to_original, +) from glin_profanity.utils.leetspeak import ( normalize_leetspeak, normalize_leetspeak_variants, @@ -91,6 +95,7 @@ def __init__(self, config: FilterConfig | None = None) -> None: self.detect_leetspeak = config.get("detect_leetspeak", False) self.leetspeak_level: LeetspeakLevel = config.get("leetspeak_level", "moderate") self.normalize_unicode_enabled = config.get("normalize_unicode", True) + self.enable_evasion_normalization = config.get("enable_evasion_normalization", True) # Caching configuration self.cache_results = config.get("cache_results", False) @@ -126,12 +131,17 @@ def _load_words(self, config: FilterConfig) -> None: # Store as set for faster lookup; track script per entry for boundary rules self.words: set[str] = {word.lower() for word in words} self.word_scripts: dict[str, WordScript] = {} + ac_words: list[str] = [] + seen_ac_keys: set[str] = set() for word in words: key = word.lower() self.word_scripts[key] = classify_word_script(word) + if key not in seen_ac_keys: + seen_ac_keys.add(key) + ac_words.append(word if self.case_sensitive else key) if self._should_use_aho_corasick(config): - self.dictionary_matcher = DictionaryAhoCorasick(list(self.words)) + self.dictionary_matcher = DictionaryAhoCorasick(ac_words) def _should_use_aho_corasick(self, config: FilterConfig) -> bool: if config.get("disable_aho_corasick"): @@ -153,7 +163,7 @@ def _debug_log(self, *args: object) -> None: def _get_normalized_variants(self, text: str) -> tuple[str, str]: """Compute normal and aggressive normalized text in one pass.""" - base = normalize_evasion(text) + base = normalize_evasion(text) if self.enable_evasion_normalization else text if self.normalize_unicode_enabled: base = normalize_unicode(base) @@ -201,6 +211,63 @@ def _get_text_variants(self, text: str, lowercase: bool) -> dict[str, str]: "aggressive": aggressive, } + def _for_each_text_variant( + self, + variants: dict[str, str], + callback: Callable[[str, bool], bool], + ) -> bool: + if callback(variants["original"], True): + return True + if variants["normalized"] != variants["original"] and callback( + variants["normalized"], False + ): + return True + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + and callback(variants["aggressive"], False) + ): + return True + return False + + def _resolve_ac_match_in_original( + self, + original_text: str, + variant_text: str, + match: DictionaryMatch, + is_original_variant: bool, + ) -> tuple[str, int, int]: + if is_original_variant and variant_text == original_text: + return match.matched_text, match.start, match.end + + span = map_variant_span_to_original( + original_text, + variant_text, + match.start, + match.end, + ) + return span.matched_text, span.start, span.end + + def _resolve_regex_match_in_original( + self, + original_text: str, + variant_text: str, + match_start: int, + match_end: int, + matched_word: str, + is_original_variant: bool, + ) -> tuple[str, int, int]: + if is_original_variant and variant_text == original_text: + return matched_word, match_start, match_end + + span = map_variant_span_to_original( + original_text, + variant_text, + match_start, + match_end, + ) + return span.matched_text, span.start, span.end + def _evaluate_severity_on_variants( self, word: str, variants: dict[str, str] ) -> SeverityLevel | None: @@ -225,11 +292,12 @@ def _collect_matches_from_variant( self, dict_word: str, variant_text: str, + original_text: str, severity: SeverityLevel, profane_words: set[str], severity_map: dict[str, SeverityLevel], matches: list[Match], - use_match_text: bool, + is_original_variant: bool, ) -> None: regex = self._get_regex(dict_word) script = self._get_word_script(dict_word) @@ -242,15 +310,22 @@ def _collect_matches_from_variant( ): continue - matched = match.group(0) if use_match_text else dict_word - profane_words.add(matched) - if matched not in severity_map: - severity_map[matched] = severity + matched_word, resolved_start, _ = self._resolve_regex_match_in_original( + original_text, + variant_text, + start, + end, + match.group(0) if is_original_variant else dict_word, + is_original_variant, + ) + profane_words.add(matched_word) + if matched_word not in severity_map: + severity_map[matched_word] = severity matches.append( { - "word": matched, - "index": start, + "word": matched_word, + "index": resolved_start, "severity": severity, } ) @@ -281,6 +356,7 @@ def _normalize_obfuscated(self, text: str) -> str: def clear_cache(self) -> None: """Clear the result cache.""" self._cache.clear() + self._regex_cache.clear() def get_cache_size(self) -> int: """Get the current cache size.""" @@ -320,6 +396,7 @@ def get_config(self) -> FilterConfig: "detect_leetspeak": self.detect_leetspeak, "leetspeak_level": self.leetspeak_level, "normalize_unicode": self.normalize_unicode_enabled, + "enable_evasion_normalization": self.enable_evasion_normalization, "cache_results": self.cache_results, "max_cache_size": self.max_cache_size, } @@ -452,19 +529,10 @@ def _is_profane_with_aho_corasick(self, value: str) -> bool: matcher = self.dictionary_matcher assert matcher is not None - if matcher.has_any_match(variants["original"], options): - return True - if variants["normalized"] != variants["original"] and matcher.has_any_match( - variants["normalized"], options - ): - return True - if ( - variants["aggressive"] != variants["normalized"] - and variants["aggressive"] != variants["original"] - and matcher.has_any_match(variants["aggressive"], options) - ): - return True - return False + def check_variant(variant_text: str, _is_original: bool) -> bool: + return matcher.has_any_match(variant_text, options) + + return self._for_each_text_variant(variants, check_variant) def _is_profane_legacy(self, value: str) -> bool: variants = self._get_text_variants(value, False) @@ -485,26 +553,19 @@ def _check_profanity_with_aho_corasick(self, text: str) -> CheckProfanityResult: matcher = self.dictionary_matcher assert matcher is not None - for match in matcher.find_matches(variants["original"], options): - profane_words.add(match.matched_text) - if match.matched_text not in severity_map: - severity_map[match.matched_text] = SeverityLevel.EXACT - - if variants["normalized"] != variants["original"]: - for match in matcher.find_matches(variants["normalized"], options): - profane_words.add(match.dict_word) - if match.dict_word not in severity_map: - severity_map[match.dict_word] = SeverityLevel.EXACT - - if ( - variants["aggressive"] != variants["normalized"] - and variants["aggressive"] != variants["original"] - ): - for match in matcher.find_matches(variants["aggressive"], options): - profane_words.add(match.dict_word) - if match.dict_word not in severity_map: - severity_map[match.dict_word] = SeverityLevel.EXACT + def collect_from_variant(variant_text: str, is_original: bool) -> bool: + for match in matcher.find_matches(variant_text, options): + matched_word, _, _ = self._resolve_ac_match_in_original( + text, variant_text, match, is_original + ) + if not matched_word: + continue + profane_words.add(matched_word) + if matched_word not in severity_map: + severity_map[matched_word] = SeverityLevel.EXACT + return False + self._for_each_text_variant(variants, collect_from_variant) return self._build_profanity_result(text, profane_words, severity_map) def _check_profanity_legacy_non_context(self, text: str) -> CheckProfanityResult: @@ -522,6 +583,7 @@ def _check_profanity_legacy_non_context(self, text: str) -> CheckProfanityResult self._collect_matches_from_variant( dict_word, variants["original"], + text, severity, profane_words_set, severity_map, @@ -535,6 +597,7 @@ def _check_profanity_legacy_non_context(self, text: str) -> CheckProfanityResult self._collect_matches_from_variant( dict_word, variants["normalized"], + text, severity, profane_words_set, severity_map, @@ -548,9 +611,16 @@ def _check_profanity_legacy_non_context(self, text: str) -> CheckProfanityResult ): severity = self._evaluate_severity(dict_word, variants["aggressive"]) if severity is not None: - profane_words_set.add(dict_word) - if dict_word not in severity_map: - severity_map[dict_word] = SeverityLevel.EXACT + self._collect_matches_from_variant( + dict_word, + variants["aggressive"], + text, + severity, + profane_words_set, + severity_map, + matches, + False, + ) result = self._build_profanity_result(text, profane_words_set, severity_map) if matches: @@ -562,13 +632,27 @@ def _check_profanity_legacy_non_context(self, text: str) -> CheckProfanityResult ) return result + def _dedupe_nested_profane_words(self, words: list[str]) -> list[str]: + return [ + word + for word in words + if not any( + other != word and is_nested_profane_span(word, other) + for other in words + ) + ] + def _build_profanity_result( self, text: str, profane_words: set[str], severity_map: dict[str, SeverityLevel], ) -> CheckProfanityResult: - profane_word_list = list(profane_words) + profane_word_list = sorted( + self._dedupe_nested_profane_words(list(profane_words)), + key=len, + reverse=True, + ) processed_text = text if self.replace_with and profane_word_list: @@ -635,7 +719,10 @@ def _record_context_aware_match( match_obj["context_score"] = context_result.context_score match_obj["reason"] = context_result.reason match_obj["is_whitelisted"] = context_result.is_whitelisted - if not self._passes_context_filter(text, matched_word, match_index): + if ( + context_result.is_whitelisted + or context_result.context_score > self.confidence_threshold + ): return seen.add(dedupe_key) @@ -657,11 +744,16 @@ def _collect_context_aware_candidates_from_ac( matcher = self.dictionary_matcher assert matcher is not None - def process_ac_match(match: DictionaryMatch, use_matched_text: bool) -> None: + def process_ac_match( + variant_text: str, match: DictionaryMatch, is_original: bool + ) -> None: + matched_word, start, _ = self._resolve_ac_match_in_original( + text, variant_text, match, is_original + ) self._record_context_aware_match( text, - match.matched_text if use_matched_text else match.dict_word, - match.start, + matched_word, + start, SeverityLevel.EXACT, profane_words, severity_map, @@ -669,19 +761,48 @@ def process_ac_match(match: DictionaryMatch, use_matched_text: bool) -> None: seen, ) - for match in matcher.find_matches(variants["original"], options): - process_ac_match(match, True) + def collect_from_variant(variant_text: str, is_original: bool) -> bool: + for match in matcher.find_matches(variant_text, options): + process_ac_match(variant_text, match, is_original) + return False - if variants["normalized"] != variants["original"]: - for match in matcher.find_matches(variants["normalized"], options): - process_ac_match(match, False) + self._for_each_text_variant(variants, collect_from_variant) - if ( - variants["aggressive"] != variants["normalized"] - and variants["aggressive"] != variants["original"] - ): - for match in matcher.find_matches(variants["aggressive"], options): - process_ac_match(match, False) + def _collect_context_aware_candidates_from_legacy_fuzzy( + self, + text: str, + variants: dict[str, str], + profane_words: list[str], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + seen: set[str], + ) -> None: + for dict_word in self.words: + if dict_word.lower() in self.ignore_words: + continue + + def collect_from_variant(variant_text: str) -> None: + if self._evaluate_severity(dict_word, variant_text) != SeverityLevel.FUZZY: + return + self._record_context_aware_match( + text, + dict_word, + 0, + SeverityLevel.FUZZY, + profane_words, + severity_map, + matches, + seen, + ) + + collect_from_variant(variants["original"]) + if variants["normalized"] != variants["original"]: + collect_from_variant(variants["normalized"]) + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + collect_from_variant(variants["aggressive"]) def _collect_context_aware_candidates_from_legacy( self, @@ -696,11 +817,24 @@ def _collect_context_aware_candidates_from_legacy( if dict_word.lower() in self.ignore_words: continue - def collect_from_variant(variant_text: str, use_match_text: bool) -> None: + def collect_from_variant(variant_text: str, is_original: bool) -> None: severity = self._evaluate_severity(dict_word, variant_text) if severity is None: return + if not self.word_boundaries and severity == SeverityLevel.FUZZY: + self._record_context_aware_match( + text, + dict_word, + 0, + severity, + profane_words, + severity_map, + matches, + seen, + ) + return + regex = self._get_regex(dict_word) script = self._get_word_script(dict_word) for match in regex.finditer(variant_text): @@ -710,10 +844,18 @@ def collect_from_variant(variant_text: str, use_match_text: bool) -> None: variant_text, start, end, script, self.word_boundaries ): continue - self._record_context_aware_match( + matched_word, resolved_start, _ = self._resolve_regex_match_in_original( text, - match.group(0) if use_match_text else dict_word, + variant_text, start, + end, + match.group(0) if is_original else dict_word, + is_original, + ) + self._record_context_aware_match( + text, + matched_word, + resolved_start, severity, profane_words, severity_map, @@ -741,7 +883,11 @@ def _build_context_aware_result( ) -> CheckProfanityResult: processed_text = text if self.replace_with and profane_words: - for word in dict.fromkeys(profane_words): + for word in sorted( + self._dedupe_nested_profane_words(list(dict.fromkeys(profane_words))), + key=len, + reverse=True, + ): processed_text = self._get_replacement_regex(word).sub( self.replace_with, processed_text ) @@ -754,7 +900,9 @@ def _build_context_aware_result( result: CheckProfanityResult = { "contains_profanity": len(profane_words) > 0, - "profane_words": list(dict.fromkeys(profane_words)), + "profane_words": sorted( + self._dedupe_nested_profane_words(list(dict.fromkeys(profane_words))) + ), "reason": ( f"Found {len(matches)} potential profanity matches" if matches @@ -767,7 +915,9 @@ def _build_context_aware_result( if self.severity_levels and severity_map: result["severity_map"] = severity_map if matches: - result["matches"] = matches + result["matches"] = sorted( + matches, key=lambda match: (match["index"], match["word"]) + ) if context_score is not None: result["context_score"] = context_score @@ -784,6 +934,10 @@ def _check_profanity_with_context_aware(self, text: str) -> CheckProfanityResult self._collect_context_aware_candidates_from_ac( text, variants, profane_words, severity_map, matches, seen ) + if not self.word_boundaries: + self._collect_context_aware_candidates_from_legacy_fuzzy( + text, variants, profane_words, severity_map, matches, seen + ) else: self._collect_context_aware_candidates_from_legacy( text, variants, profane_words, severity_map, matches, seen @@ -801,35 +955,28 @@ def _has_context_aware_ac_match(self, text: str, variants: dict[str, str]) -> bo matcher = self.dictionary_matcher assert matcher is not None - def has_flagged_match(variant_text: str, use_matched_text: bool) -> bool: + def has_flagged_match(variant_text: str, is_original: bool) -> bool: for match in matcher.find_matches(variant_text, options): - matched_word = match.matched_text if use_matched_text else match.dict_word - if self._passes_context_filter(text, matched_word, match.start): + matched_word, start, _ = self._resolve_ac_match_in_original( + text, variant_text, match, is_original + ) + if self._passes_context_filter(text, matched_word, start): return True return False - if has_flagged_match(variants["original"], True): - return True - if variants["normalized"] != variants["original"] and has_flagged_match( - variants["normalized"], False - ): - return True - if ( - variants["aggressive"] != variants["normalized"] - and variants["aggressive"] != variants["original"] - and has_flagged_match(variants["aggressive"], False) - ): - return True - return False + return self._for_each_text_variant(variants, has_flagged_match) def _has_context_aware_legacy_match_for_word( self, text: str, variants: dict[str, str], dict_word: str ) -> bool: - def check_variant(variant_text: str, use_match_text: bool) -> bool: + def check_variant(variant_text: str, is_original: bool) -> bool: severity = self._evaluate_severity(dict_word, variant_text) if severity is None: return False + if not self.word_boundaries and severity == SeverityLevel.FUZZY: + return self._passes_context_filter(text, dict_word, 0) + regex = self._get_regex(dict_word) script = self._get_word_script(dict_word) for match in regex.finditer(variant_text): @@ -839,8 +986,15 @@ def check_variant(variant_text: str, use_match_text: bool) -> bool: variant_text, start, end, script, self.word_boundaries ): continue - matched_word = match.group(0) if use_match_text else dict_word - if self._passes_context_filter(text, matched_word, start): + matched_word, resolved_start, _ = self._resolve_regex_match_in_original( + text, + variant_text, + start, + end, + match.group(0) if is_original else dict_word, + is_original, + ) + if self._passes_context_filter(text, matched_word, resolved_start): return True return False @@ -858,6 +1012,40 @@ def check_variant(variant_text: str, use_match_text: bool) -> bool: return True return False + def _has_context_aware_legacy_fuzzy_match( + self, text: str, variants: dict[str, str] + ) -> bool: + for dict_word in self.words: + if dict_word.lower() in self.ignore_words: + continue + if self._has_context_aware_legacy_fuzzy_match_for_word( + text, variants, dict_word + ): + return True + return False + + def _has_context_aware_legacy_fuzzy_match_for_word( + self, text: str, variants: dict[str, str], dict_word: str + ) -> bool: + def check_variant(variant_text: str) -> bool: + if self._evaluate_severity(dict_word, variant_text) != SeverityLevel.FUZZY: + return False + return self._passes_context_filter(text, dict_word, 0) + + if check_variant(variants["original"]): + return True + if variants["normalized"] != variants["original"] and check_variant( + variants["normalized"] + ): + return True + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + and check_variant(variants["aggressive"]) + ): + return True + return False + def _has_context_aware_legacy_match(self, text: str, variants: dict[str, str]) -> bool: for dict_word in self.words: if dict_word.lower() in self.ignore_words: @@ -869,7 +1057,13 @@ def _has_context_aware_legacy_match(self, text: str, variants: dict[str, str]) - def _is_profane_with_context_aware(self, value: str) -> bool: variants = self._get_text_variants(value, False) if self.dictionary_matcher: - return self._has_context_aware_ac_match(value, variants) + if self._has_context_aware_ac_match(value, variants): + return True + if not self.word_boundaries and self._has_context_aware_legacy_fuzzy_match( + value, variants + ): + return True + return False return self._has_context_aware_legacy_match(value, variants) def is_profane(self, value: str) -> bool: diff --git a/packages/py/glin_profanity/nlp/context_analyzer.py b/packages/py/glin_profanity/nlp/context_analyzer.py index 0a69cfd..f899cba 100644 --- a/packages/py/glin_profanity/nlp/context_analyzer.py +++ b/packages/py/glin_profanity/nlp/context_analyzer.py @@ -234,8 +234,8 @@ def __init__(self, config: ContextConfig) -> None: def analyze_context( self, text: str, match_word: str, match_index: int ) -> ContextAnalysisResult: - words = self._tokenize(text) - match_word_index = self._find_word_index(words, match_index) + tokens = self._tokenize_with_spans(text) + match_word_index = self._find_word_index(tokens, match_index) if match_word_index == -1: return ContextAnalysisResult( @@ -245,8 +245,8 @@ def analyze_context( ) start_index = max(0, match_word_index - self.context_window) - end_index = min(len(words), match_word_index + self.context_window + 1) - context_words = words[start_index:end_index] + end_index = min(len(tokens), match_word_index + self.context_window + 1) + context_words = [token["word"] for token in tokens[start_index:end_index]] context_text = " ".join(context_words).lower() phrase_result = self._check_phrase_context(context_text, match_word) @@ -319,17 +319,39 @@ def _generate_reason(self, score: float, context_words: list[str]) -> str: return f"Negative context detected{details} - likely profanity" return "Neutral context - uncertain classification" - def _tokenize(self, text: str) -> list[str]: - normalized = re.sub(r"[^A-Za-z0-9_\s]", " ", text.lower()) - return [word for word in normalized.split() if word] - - def _find_word_index(self, words: list[str], char_index: int) -> int: - current_pos = 0 - for i, word in enumerate(words): - if current_pos >= char_index: + def _tokenize_with_spans(self, text: str) -> list[dict[str, int | str]]: + tokens: list[dict[str, int | str]] = [] + for match in re.finditer(r"\S+", text): + raw = match.group(0) + normalized = re.sub(r"[^\w]", "", raw.lower(), flags=re.UNICODE) + if normalized: + tokens.append( + { + "word": normalized, + "start": match.start(), + "end": match.end(), + } + ) + return tokens + + def _find_word_index( + self, tokens: list[dict[str, int | str]], char_index: int + ) -> int: + if not tokens: + return -1 + + for i, token in enumerate(tokens): + start = int(token["start"]) + end = int(token["end"]) + if char_index >= start and char_index < end: + return i + if char_index < start: return max(0, i - 1) - current_pos += len(word) + 1 - return len(words) - 1 + + return len(tokens) - 1 + + def _tokenize(self, text: str) -> list[str]: + return [str(token["word"]) for token in self._tokenize_with_spans(text)] def _calculate_sentiment_score( self, context_words: list[str], match_position: int diff --git a/packages/py/glin_profanity/types/types.py b/packages/py/glin_profanity/types/types.py index 14a78e6..7e3f8dd 100644 --- a/packages/py/glin_profanity/types/types.py +++ b/packages/py/glin_profanity/types/types.py @@ -109,6 +109,9 @@ class FilterConfig(ContextAwareConfig, total=False): # Performance options disable_aho_corasick: bool # Force legacy regex path (testing / debugging) + # Evasion normalization + enable_evasion_normalization: bool # HTML/separators/masking pipeline + class FilteredProfanityResult(TypedDict): """Result with minimum severity filtering.""" diff --git a/packages/py/glin_profanity/utils/unicode.py b/packages/py/glin_profanity/utils/unicode.py index 38ab56e..1d3385b 100644 --- a/packages/py/glin_profanity/utils/unicode.py +++ b/packages/py/glin_profanity/utils/unicode.py @@ -193,6 +193,11 @@ def convert_full_width(text: str) -> str: return "".join(result) +def homoglyph_to_ascii(char: str) -> str: + """Return the ASCII lookalike for a single homoglyph character.""" + return HOMOGLYPHS.get(char, char) + + def convert_homoglyphs(text: str) -> str: """ Convert homoglyph characters to their ASCII equivalents. diff --git a/packages/py/glin_profanity/utils/variant_mapping.py b/packages/py/glin_profanity/utils/variant_mapping.py new file mode 100644 index 0000000..85b6376 --- /dev/null +++ b/packages/py/glin_profanity/utils/variant_mapping.py @@ -0,0 +1,167 @@ +"""Map span positions from normalized variant text back to the original.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from glin_profanity.utils.unicode import homoglyph_to_ascii + +_LEET_TO_ASCII = { + "@": "a", + "$": "s", + "!": "i", + "1": "i", + "0": "o", + "3": "e", + "4": "a", + "5": "s", + "7": "t", + "8": "b", + "9": "g", +} + +_SKIPPABLE_ORIGINAL_CHARS = {"*", ".", "_", "-", " "} + + +@dataclass(frozen=True) +class OriginalSpan: + start: int + end: int + matched_text: str + + +def _chars_equal(left: str, right: str) -> bool: + return left == right or left.lower() == right.lower() + + +def _chars_align(original_char: str, variant_char: str) -> bool: + if _chars_equal(original_char, variant_char): + return True + + mapped = _LEET_TO_ASCII.get(original_char) or _LEET_TO_ASCII.get( + original_char.lower() + ) + if mapped is not None and _chars_equal(mapped, variant_char): + return True + + homoglyph = homoglyph_to_ascii(original_char) + if homoglyph != original_char and _chars_equal(homoglyph, variant_char): + return True + + return False + + +def _is_skippable_original_char(char: str) -> bool: + return char in _SKIPPABLE_ORIGINAL_CHARS + + +def _fallback_span( + original: str, variant: str, variant_start: int, variant_end: int +) -> OriginalSpan: + needle = variant[variant_start:variant_end] + if not needle: + return OriginalSpan(start=0, end=0, matched_text="") + + idx = original.lower().find(needle.lower()) + if idx >= 0: + return OriginalSpan( + start=idx, + end=idx + len(needle), + matched_text=original[idx : idx + len(needle)], + ) + + start = min(variant_start, len(original)) + end = min(variant_end, len(original)) + return OriginalSpan(start=start, end=end, matched_text=original[start:end]) + + +def map_variant_span_to_original( + original: str, + variant: str, + variant_start: int, + variant_end: int, +) -> OriginalSpan: + """Locate the original span for ``variant[variant_start:variant_end]``.""" + if variant_start >= variant_end: + return OriginalSpan(start=0, end=0, matched_text="") + + if original == variant: + return OriginalSpan( + start=variant_start, + end=variant_end, + matched_text=original[variant_start:variant_end], + ) + + original_index = 0 + variant_index = 0 + orig_start = -1 + orig_end = -1 + + while variant_index < len(variant) and original_index <= len(original): + if variant_index == variant_start and orig_start == -1: + orig_start = original_index + if variant_index == variant_end: + orig_end = original_index + break + + if original_index >= len(original): + variant_index += 1 + continue + + variant_char = variant[variant_index] + original_char = original[original_index] + + if _chars_align(original_char, variant_char): + variant_index += 1 + original_index += 1 + continue + + if _is_skippable_original_char(original_char): + original_index += 1 + continue + + original_index += 1 + + if orig_start == -1: + orig_start = 0 + if orig_end == -1: + orig_end = len(original) + + matched_text = original[orig_start:orig_end] + if not matched_text or orig_start >= orig_end: + return _fallback_span(original, variant, variant_start, variant_end) + + return OriginalSpan( + start=orig_start, + end=orig_end, + matched_text=matched_text, + ) + + +_NESTED_WORD_BOUNDARY = re.compile(r"\w") + + +def is_nested_profane_span(shorter: str, longer: str) -> bool: + """True when shorter is a word-bounded substring of longer.""" + if len(longer) <= len(shorter): + return False + + search_from = 0 + while search_from <= len(longer) - len(shorter): + idx = longer.find(shorter, search_from) + if idx == -1: + return False + + before_ok = idx == 0 or not _NESTED_WORD_BOUNDARY.match(longer[idx - 1]) + after_idx = idx + len(shorter) + after_ok = after_idx == len(longer) or not _NESTED_WORD_BOUNDARY.match( + longer[after_idx] + ) + + if before_ok and after_ok: + return True + + search_from = idx + 1 + + return False diff --git a/packages/py/tests/test_filter_pool_export.py b/packages/py/tests/test_filter_pool_export.py new file mode 100644 index 0000000..c0a33c8 --- /dev/null +++ b/packages/py/tests/test_filter_pool_export.py @@ -0,0 +1,22 @@ +"""Tests for filter pool public exports.""" + +from glin_profanity import clear_filter_pool, create_filter_config, get_pooled_filter +from glin_profanity.filters.filter import Filter + + +class TestFilterPoolExport: + def setup_method(self) -> None: + clear_filter_pool() + + def teardown_method(self) -> None: + clear_filter_pool() + + def test_get_pooled_filter_returns_filter(self) -> None: + config = create_filter_config({"languages": ["english"]}) + assert isinstance(get_pooled_filter(config), Filter) + + def test_reuses_same_instance(self) -> None: + config = create_filter_config({"languages": ["english"]}) + first = get_pooled_filter(config) + second = get_pooled_filter(config) + assert first is second diff --git a/packages/py/tests/test_variant_mapping.py b/packages/py/tests/test_variant_mapping.py new file mode 100644 index 0000000..e603d55 --- /dev/null +++ b/packages/py/tests/test_variant_mapping.py @@ -0,0 +1,54 @@ +"""Tests for variant-to-original span mapping.""" + +from glin_profanity.utils.variant_mapping import ( + is_nested_profane_span, + map_variant_span_to_original, +) + + +class TestVariantMapping: + def test_maps_collapsed_separators(self) -> None: + original = "say f.u.c.k off" + variant = "say fuck off" + span = map_variant_span_to_original(original, variant, 4, 8) + assert span.matched_text == "f.u.c.k" + assert original[span.start : span.end] == "f.u.c.k" + + def test_maps_leetspeak_substitutions(self) -> None: + span = map_variant_span_to_original("@ss", "ass", 0, 3) + assert span.matched_text == "@ss" + + def test_maps_repeated_character_collapse(self) -> None: + span = map_variant_span_to_original("fuuuuuck", "fuck", 0, 4) + assert span.matched_text == "fuuuuuck" + + def test_maps_asterisk_masking(self) -> None: + span = map_variant_span_to_original("holy f***", "holy fuck", 5, 9) + assert span.matched_text == "f***" + + def test_maps_unicode_homoglyphs(self) -> None: + original = "fυck you" + variant = "fuck you" + span = map_variant_span_to_original(original, variant, 0, 4) + assert span.matched_text == "fυck" + + def test_maps_lowercased_original_tier_to_mixed_case(self) -> None: + original = "What a FUCK" + variant = "what a fuck" + span = map_variant_span_to_original(original, variant, 7, 11) + assert span.matched_text == "FUCK" + + def test_identity_when_texts_match(self) -> None: + text = "plain fuck here" + span = map_variant_span_to_original(text, text, 6, 10) + assert span.matched_text == "fuck" + assert span.start == 6 + assert span.end == 10 + + +class TestNestedProfaneSpan: + def test_detects_nested_spans(self) -> None: + assert is_nested_profane_span("sh!t", "piece of sh!t") + + def test_ignores_ass_inside_classic(self) -> None: + assert not is_nested_profane_span("ass", "classic") diff --git a/tests/cross_language_parity_test.py b/tests/cross_language_parity_test.py index ca0e723..a306c2c 100644 --- a/tests/cross_language_parity_test.py +++ b/tests/cross_language_parity_test.py @@ -47,6 +47,7 @@ def python_config_to_js(config: dict[str, Any]) -> dict[str, Any]: "cache_results": "cacheResults", "max_cache_size": "maxCacheSize", "disable_aho_corasick": "disableAhoCorasick", + "enable_evasion_normalization": "enableEvasionNormalization", "domain_whitelists": "domainWhitelists", "log_profanity": "logProfanity", } From 49361391b544c52df340f0fa522f5fdfc9047522 Mon Sep 17 00:00:00 2001 From: wlike Date: Sat, 27 Jun 2026 18:46:43 +0800 Subject: [PATCH 03/11] fix: align profane_words collection with normalized-first fallback and result parity Ensure check_profanity returns original-text spans via variant mapping and tier fallback, while keeping normalized-tier priority to avoid nested false positives. Unify contains/reason handling across AC, legacy, and context-aware paths in Python and JavaScript. Co-authored-by: Cursor --- benchmarks/compare_csv_packages.py | 173 ++++++ packages/js/src/filters/Filter.ts | 506 +++++++++++------- packages/js/src/utils/leetspeak.ts | 2 + packages/js/src/utils/variantMapping.ts | 178 +++++- packages/js/tests/context-aware.test.ts | 31 ++ packages/js/tests/variant-mapping.test.ts | 21 +- packages/py/glin_profanity/filters/filter.py | 485 ++++++++++------- .../glin_profanity/utils/variant_mapping.py | 147 ++++- .../py/tests/test_profane_words_collection.py | 110 ++++ packages/py/tests/test_variant_mapping.py | 17 + 10 files changed, 1236 insertions(+), 434 deletions(-) create mode 100644 benchmarks/compare_csv_packages.py create mode 100644 packages/py/tests/test_profane_words_collection.py diff --git a/benchmarks/compare_csv_packages.py b/benchmarks/compare_csv_packages.py new file mode 100644 index 0000000..bc673f7 --- /dev/null +++ b/benchmarks/compare_csv_packages.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Compare glin_profanity 3.4.0 vs feat-performance-opt on CSV text column.""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path + +import pandas as pd + +CSV_PATH = Path( + "/Users/wlike/Downloads/请求明细(safetyScore_ge_0.6)_2026-06-24_10_56_28.csv" +) +PACKAGE_V340 = Path("/Users/wlike/Downloads/glin_profanity-3.4.0") +PACKAGE_OPT = Path("/Users/wlike/Documents/learn/glin-profanity/packages/py") +OUTPUT_XLSX = Path( + "/Users/wlike/Downloads/glin_profanity_comparison_2026-06-24.xlsx" +) + +FILTER_CONFIG = { + "all_languages": True, + "detect_leetspeak": True, + "normalize_unicode": True, +} + + +def unload_glin_profanity() -> None: + for name in list(sys.modules): + if name == "glin_profanity" or name.startswith("glin_profanity."): + del sys.modules[name] + + +def load_filter(package_parent: Path): + unload_glin_profanity() + parent = str(package_parent.resolve()) + sys.path = [p for p in sys.path if Path(p).resolve() != package_parent.resolve()] + sys.path.insert(0, parent) + from glin_profanity.filters.filter import Filter + + return Filter(FILTER_CONFIG) + + +def run_batch( + label: str, package_parent: Path, texts: list[str] +) -> tuple[list[bool], list[str], list[frozenset[str]], list[float]]: + print(f"Loading Filter from {package_parent} ({label})...") + t0 = time.perf_counter() + filt = load_filter(package_parent) + init_ms = (time.perf_counter() - t0) * 1000 + print(f" init: {init_ms:.1f} ms, words: {filt.get_word_count()}") + + contains_list: list[bool] = [] + words_list: list[str] = [] + words_sets: list[frozenset[str]] = [] + time_list: list[float] = [] + + total = len(texts) + for i, text in enumerate(texts): + if i > 0 and i % 1000 == 0: + print(f" [{label}] {i}/{total}...") + start = time.perf_counter() + result = filt.check_profanity(text if isinstance(text, str) else str(text)) + elapsed_ms = (time.perf_counter() - start) * 1000 + contains_list.append(bool(result.get("contains_profanity", False))) + words = result.get("profane_words") or [] + word_set = frozenset(words) + words_sets.append(word_set) + words_list.append("; ".join(words)) + time_list.append(elapsed_ms) + + return contains_list, words_list, words_sets, time_list + + +def classify_results_match( + v340_contains: bool, + opt_contains: bool, + v340_words: frozenset[str], + opt_words: frozenset[str], +) -> str: + if v340_contains != opt_contains: + return "false" + if v340_words == opt_words: + return "true" + return "half" + + +def main() -> None: + if not CSV_PATH.exists(): + raise SystemExit(f"CSV not found: {CSV_PATH}") + + print(f"Reading {CSV_PATH}...") + df = pd.read_csv(CSV_PATH, encoding="utf-8") + if "text" not in df.columns: + raise SystemExit(f"Missing 'text' column. Columns: {list(df.columns)}") + + texts = df["text"].fillna("").astype(str).tolist() + print(f"Rows: {len(texts)}") + + v340_contains, v340_words, v340_word_sets, v340_times = run_batch( + "3.4.0", PACKAGE_V340, texts + ) + opt_contains, opt_words, opt_word_sets, opt_times = run_batch( + "feat-opt", PACKAGE_OPT, texts + ) + + df["v340_contains_profanity"] = v340_contains + df["v340_profane_words"] = v340_words + df["v340_time_ms"] = v340_times + df["opt_contains_profanity"] = opt_contains + df["opt_profane_words"] = opt_words + df["opt_time_ms"] = opt_times + df["results_match"] = [ + classify_results_match( + v340_contains[i], opt_contains[i], v340_word_sets[i], opt_word_sets[i] + ) + for i in range(len(texts)) + ] + + v340_avg = sum(v340_times) / len(v340_times) if v340_times else 0.0 + opt_avg = sum(opt_times) / len(opt_times) if opt_times else 0.0 + v340_total = sum(v340_times) + opt_total = sum(opt_times) + true_count = int((df["results_match"] == "true").sum()) + half_count = int((df["results_match"] == "half").sum()) + false_count = int((df["results_match"] == "false").sum()) + v340_flagged = sum(v340_contains) + opt_flagged = sum(opt_contains) + total_rows = len(df) + + summary = pd.DataFrame( + [ + {"metric": "total_rows", "value": total_rows}, + {"metric": "v340_avg_time_ms", "value": round(v340_avg, 4)}, + {"metric": "opt_avg_time_ms", "value": round(opt_avg, 4)}, + {"metric": "v340_total_time_ms", "value": round(v340_total, 2)}, + {"metric": "opt_total_time_ms", "value": round(opt_total, 2)}, + {"metric": "speedup_vs_v340", "value": round(v340_avg / opt_avg, 4) if opt_avg else None}, + {"metric": "v340_flagged_count", "value": v340_flagged}, + {"metric": "opt_flagged_count", "value": opt_flagged}, + {"metric": "results_match_true_count", "value": true_count}, + {"metric": "results_match_half_count", "value": half_count}, + {"metric": "results_match_false_count", "value": false_count}, + { + "metric": "results_match_true_rate_pct", + "value": round(100 * true_count / total_rows, 4) if total_rows else 0, + }, + { + "metric": "results_match_half_rate_pct", + "value": round(100 * half_count / total_rows, 4) if total_rows else 0, + }, + { + "metric": "results_match_false_rate_pct", + "value": round(100 * false_count / total_rows, 4) if total_rows else 0, + }, + ] + ) + + print(f"Writing {OUTPUT_XLSX}...") + with pd.ExcelWriter(OUTPUT_XLSX, engine="openpyxl") as writer: + df.to_excel(writer, sheet_name="明细", index=False) + summary.to_excel(writer, sheet_name="汇总", index=False) + + print("Done.") + print(f" v340 avg: {v340_avg:.4f} ms | opt avg: {opt_avg:.4f} ms") + print( + f" flagged: v340={v340_flagged} opt={opt_flagged} | " + f"match true={true_count} half={half_count} false={false_count}" + ) + + +if __name__ == "__main__": + main() diff --git a/packages/js/src/filters/Filter.ts b/packages/js/src/filters/Filter.ts index 9a77b74..b569e24 100644 --- a/packages/js/src/filters/Filter.ts +++ b/packages/js/src/filters/Filter.ts @@ -4,7 +4,7 @@ import { ContextAnalyzer } from '../nlp/contextAnalyzer'; import { DictionaryAhoCorasick, type DictionaryMatch } from './dictionaryAhoCorasick'; import { normalizeLeetspeak, normalizeLeetspeakVariants } from '../utils/leetspeak'; import { normalizeEvasion } from '../utils/evasion'; -import { mapVariantSpanToOriginal, isNestedProfaneSpan } from '../utils/variantMapping'; +import { mapVariantSpanToOriginal, isNestedProfaneSpan, dedupeProfaneSpansByOverlap } from '../utils/variantMapping'; import { normalizeUnicode } from '../utils/unicode'; import { classifyWordScript, @@ -256,16 +256,8 @@ class Filter { originalText: string, variantText: string, match: DictionaryMatch, - isOriginalVariant: boolean, + _isOriginalVariant: boolean, ): { matchedWord: string; start: number; end: number } { - if (isOriginalVariant && variantText === originalText) { - return { - matchedWord: match.matchedText, - start: match.start, - end: match.end, - }; - } - const span = mapVariantSpanToOriginal( originalText, variantText, @@ -284,13 +276,9 @@ class Filter { variantText: string, matchStart: number, matchEnd: number, - matchedWord: string, - isOriginalVariant: boolean, + _matchedWord: string, + _isOriginalVariant: boolean, ): { matchedWord: string; start: number; end: number } { - if (isOriginalVariant && variantText === originalText) { - return { matchedWord, start: matchStart, end: matchEnd }; - } - const span = mapVariantSpanToOriginal( originalText, variantText, @@ -354,12 +342,80 @@ class Filter { return undefined; } + private profaneWordsFromSpans( + spans: Array<[string, number, number]>, + severityMap: Record, + ): { profaneWords: Set; severityMap: Record } { + const deduped = dedupeProfaneSpansByOverlap(spans); + const profaneWords = new Set(); + const resolvedSeverityMap: Record = {}; + + for (const [word] of deduped) { + if (!word) { + continue; + } + profaneWords.add(word); + resolvedSeverityMap[word] = severityMap[word] ?? SeverityLevel.EXACT; + } + + return { profaneWords, severityMap: resolvedSeverityMap }; + } + + private collectProfaneSpansFromVariantAc( + text: string, + variantText: string, + ): Array<[string, number, number]> { + const matcher = this.dictionaryMatcher; + if (!matcher) { + return []; + } + + const options = this.getDictionarySearchOptions(); + const spans: Array<[string, number, number]> = []; + + for (const match of matcher.findMatches(variantText, options)) { + const resolved = this.resolveAcMatchInOriginal(text, variantText, match, false); + if (resolved.matchedWord) { + spans.push([resolved.matchedWord, resolved.start, resolved.end]); + } + } + + return spans; + } + + private collectProfaneSpansAc( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + containsProfanity = false, + ): Array<[string, number, number]> { + let spans = this.collectProfaneSpansFromVariantAc(text, variants.normalized); + if (spans.length > 0 || !containsProfanity) { + return dedupeProfaneSpansByOverlap(spans); + } + + if (variants.original !== variants.normalized) { + spans = this.collectProfaneSpansFromVariantAc(text, variants.original); + if (spans.length > 0) { + return dedupeProfaneSpansByOverlap(spans); + } + } + + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original + ) { + spans = this.collectProfaneSpansFromVariantAc(text, variants.aggressive); + } + + return dedupeProfaneSpansByOverlap(spans); + } + private collectMatchesFromVariant( dictWord: string, variantText: string, originalText: string, severity: SeverityLevel, - profaneWords: Set, + profaneSpans: Array<[string, number, number]>, severityMap: Record, isOriginalVariant: boolean, ): void { @@ -380,7 +436,10 @@ class Filter { isOriginalVariant ? match[0] : dictWord, isOriginalVariant, ); - profaneWords.add(resolved.matchedWord); + if (!resolved.matchedWord) { + continue; + } + profaneSpans.push([resolved.matchedWord, resolved.start, resolved.end]); if (severityMap[resolved.matchedWord] === undefined) { severityMap[resolved.matchedWord] = severity; } @@ -794,89 +853,102 @@ class Filter { private checkProfanityWithAhoCorasick(text: string): CheckProfanityResult { const variants = this.getTextVariants(text, true); - const profaneWords = new Set(); const severityMap: Record = {}; - const options = this.getDictionarySearchOptions(); - const matcher = this.dictionaryMatcher!; + const containsProfanity = this.isProfaneWithAhoCorasick(text); + const profaneSpans = this.collectProfaneSpansAc(text, variants, containsProfanity); + const resolved = this.profaneWordsFromSpans(profaneSpans, severityMap); - this.forEachTextVariant(variants, (variantText, isOriginal) => { - for (const match of matcher.findMatches(variantText, options)) { - const resolved = this.resolveAcMatchInOriginal( - text, - variantText, - match, - isOriginal, - ); - if (!resolved.matchedWord) { - continue; - } - profaneWords.add(resolved.matchedWord); - if (severityMap[resolved.matchedWord] === undefined) { - severityMap[resolved.matchedWord] = SeverityLevel.EXACT; - } - } - return false; - }); - - return this.buildProfanityResult(text, profaneWords, severityMap); + return this.buildProfanityResult( + text, + resolved.profaneWords, + resolved.severityMap, + containsProfanity, + ); } - private checkProfanityLegacyNonContext(text: string): CheckProfanityResult { - const variants = this.getTextVariants(text, true); - const profaneWords = new Set(); - const severityMap: Record = {}; - + private collectLegacySpansFromVariant( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + variantKey: 'original' | 'normalized' | 'aggressive', + isOriginalVariant: boolean, + profaneSpans: Array<[string, number, number]>, + severityMap: Record, + ): void { + const variantText = variants[variantKey]; for (const dictWord of this.words.keys()) { if (this.ignoreWords.has(dictWord.toLowerCase())) continue; - let severity = this.evaluateSeverity(dictWord, variants.original); + const severity = this.evaluateSeverity(dictWord, variantText); if (severity !== undefined) { this.collectMatchesFromVariant( dictWord, - variants.original, + variantText, text, severity, - profaneWords, + profaneSpans, severityMap, - true, + isOriginalVariant, ); } + } + } - if (variants.normalized !== variants.original) { - severity = this.evaluateSeverity(dictWord, variants.normalized); - if (severity !== undefined) { - this.collectMatchesFromVariant( - dictWord, - variants.normalized, - text, - severity, - profaneWords, - severityMap, - false, - ); - } + private checkProfanityLegacyNonContext(text: string): CheckProfanityResult { + const variants = this.getTextVariants(text, true); + const profaneSpans: Array<[string, number, number]> = []; + const severityMap: Record = {}; + let containsProfanity = false; + + for (const dictWord of this.words.keys()) { + if (this.ignoreWords.has(dictWord.toLowerCase())) continue; + + if (this.evaluateSeverityOnVariants(dictWord, variants) !== undefined) { + containsProfanity = true; } + } + this.collectLegacySpansFromVariant( + text, + variants, + 'normalized', + false, + profaneSpans, + severityMap, + ); + if (profaneSpans.length === 0 && containsProfanity) { + if (variants.original !== variants.normalized) { + this.collectLegacySpansFromVariant( + text, + variants, + 'original', + true, + profaneSpans, + severityMap, + ); + } if ( + profaneSpans.length === 0 && variants.aggressive !== variants.normalized && variants.aggressive !== variants.original ) { - severity = this.evaluateSeverity(dictWord, variants.aggressive); - if (severity !== undefined) { - this.collectMatchesFromVariant( - dictWord, - variants.aggressive, - text, - severity, - profaneWords, - severityMap, - false, - ); - } + this.collectLegacySpansFromVariant( + text, + variants, + 'aggressive', + false, + profaneSpans, + severityMap, + ); } } - return this.buildProfanityResult(text, profaneWords, severityMap); + const resolved = this.profaneWordsFromSpans(profaneSpans, severityMap); + return this.buildProfanityResult( + text, + resolved.profaneWords, + resolved.severityMap, + containsProfanity, + ); } private recordContextAwareMatch( @@ -932,31 +1004,76 @@ class Filter { severityMap: Record, matches: Match[], seen: Set, + variantKey: 'original' | 'normalized' | 'aggressive' = 'normalized', ): void { const options = this.getDictionarySearchOptions(); const matcher = this.dictionaryMatcher!; + const variantText = variants[variantKey]; - this.forEachTextVariant(variants, (variantText, isOriginal) => { - for (const match of matcher.findMatches(variantText, options)) { - const resolved = this.resolveAcMatchInOriginal( - text, - variantText, - match, - isOriginal, - ); - this.recordContextAwareMatch( - text, - resolved.matchedWord, - resolved.start, - SeverityLevel.EXACT, - profaneWords, - severityMap, - matches, - seen, - ); - } - return false; - }); + for (const match of matcher.findMatches(variantText, options)) { + const resolved = this.resolveAcMatchInOriginal(text, variantText, match, false); + this.recordContextAwareMatch( + text, + resolved.matchedWord, + resolved.start, + SeverityLevel.EXACT, + profaneWords, + severityMap, + matches, + seen, + ); + } + } + + private collectContextAwareCandidatesFromAcWithFallback( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + profaneWords: string[], + severityMap: Record, + matches: Match[], + seen: Set, + ): void { + this.collectContextAwareCandidatesFromAc( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + ); + if (profaneWords.length > 0) { + return; + } + + if (variants.original !== variants.normalized) { + this.collectContextAwareCandidatesFromAc( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + 'original', + ); + } + if (profaneWords.length > 0) { + return; + } + + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original + ) { + this.collectContextAwareCandidatesFromAc( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + 'aggressive', + ); + } } private collectContextAwareCandidatesFromLegacyFuzzy( @@ -972,34 +1089,19 @@ class Filter { continue; } - const collectFromVariant = (variantText: string) => { - if (this.evaluateSeverity(dictWord, variantText) !== SeverityLevel.FUZZY) { - return; - } - this.recordContextAwareMatch( - text, - dictWord, - 0, - SeverityLevel.FUZZY, - profaneWords, - severityMap, - matches, - seen, - ); - }; - - collectFromVariant(variants.original); - - if (variants.normalized !== variants.original) { - collectFromVariant(variants.normalized); - } - - if ( - variants.aggressive !== variants.normalized && - variants.aggressive !== variants.original - ) { - collectFromVariant(variants.aggressive); + if (this.evaluateSeverity(dictWord, variants.normalized) !== SeverityLevel.FUZZY) { + continue; } + this.recordContextAwareMatch( + text, + dictWord, + 0, + SeverityLevel.FUZZY, + profaneWords, + severityMap, + matches, + seen, + ); } } @@ -1016,70 +1118,53 @@ class Filter { continue; } - const collectFromVariant = ( - variantText: string, - isOriginal: boolean, - ) => { - const severity = this.evaluateSeverity(dictWord, variantText); - if (severity === undefined) { - return; - } - - if (!this.wordBoundaries && severity === SeverityLevel.FUZZY) { - this.recordContextAwareMatch( - text, - dictWord, - 0, - severity, - profaneWords, - severityMap, - matches, - seen, - ); - return; - } - - const regex = this.getRegex(dictWord); - const script = this.getWordScript(dictWord); - let match: RegExpExecArray | null; - while ((match = regex.exec(variantText)) !== null) { - const start = match.index; - const end = match.index + match[0].length; - if (!matchHasWordBoundary(variantText, start, end, script, this.wordBoundaries)) { - continue; - } - const resolved = this.resolveRegexMatchInOriginal( - text, - variantText, - start, - end, - isOriginal ? match[0] : dictWord, - isOriginal, - ); - this.recordContextAwareMatch( - text, - resolved.matchedWord, - resolved.start, - severity, - profaneWords, - severityMap, - matches, - seen, - ); - } - }; - - collectFromVariant(variants.original, true); + const severity = this.evaluateSeverity(dictWord, variants.normalized); + if (severity === undefined) { + continue; + } - if (variants.normalized !== variants.original) { - collectFromVariant(variants.normalized, false); + if (!this.wordBoundaries && severity === SeverityLevel.FUZZY) { + this.recordContextAwareMatch( + text, + dictWord, + 0, + severity, + profaneWords, + severityMap, + matches, + seen, + ); + continue; } - if ( - variants.aggressive !== variants.normalized && - variants.aggressive !== variants.original - ) { - collectFromVariant(variants.aggressive, false); + const regex = this.getRegex(dictWord); + const script = this.getWordScript(dictWord); + const variantText = variants.normalized; + let match: RegExpExecArray | null; + while ((match = regex.exec(variantText)) !== null) { + const start = match.index; + const end = match.index + match[0].length; + if (!matchHasWordBoundary(variantText, start, end, script, this.wordBoundaries)) { + continue; + } + const resolved = this.resolveRegexMatchInOriginal( + text, + variantText, + start, + end, + dictWord, + false, + ); + this.recordContextAwareMatch( + text, + resolved.matchedWord, + resolved.start, + severity, + profaneWords, + severityMap, + matches, + seen, + ); } } } @@ -1089,14 +1174,15 @@ class Filter { profaneWords: string[], severityMap: Record, matches: Match[], + containsProfanity?: boolean, ): CheckProfanityResult { + const profaneWordList = this.dedupeNestedProfaneWords( + Array.from(new Set(profaneWords)), + ).sort((a, b) => b.length - a.length || a.localeCompare(b)); let processedText = text; - if (this.replaceWith && profaneWords.length > 0) { - const uniqueWords = this.dedupeNestedProfaneWords( - Array.from(new Set(profaneWords)), - ).sort((a, b) => b.length - a.length); - for (const word of uniqueWords) { + if (this.replaceWith && profaneWordList.length > 0) { + for (const word of profaneWordList) { processedText = processedText.replace( this.getReplacementRegex(word), this.replaceWith, @@ -1113,9 +1199,14 @@ class Filter { contextScore = totalScore / matches.length; } + const flagged = + containsProfanity !== undefined + ? containsProfanity + : profaneWordList.length > 0; + return { - containsProfanity: profaneWords.length > 0, - profaneWords: this.dedupeNestedProfaneWords(Array.from(new Set(profaneWords))).sort(), + containsProfanity: flagged, + profaneWords: profaneWordList, processedText: this.replaceWith ? processedText : undefined, severityMap: this.severityLevels && Object.keys(severityMap).length > 0 @@ -1126,22 +1217,22 @@ class Filter { ? [...matches].sort((a, b) => a.index - b.index || a.word.localeCompare(b.word)) : undefined, contextScore, - reason: - matches.length > 0 - ? `Found ${matches.length} potential profanity matches` - : 'No profanity detected', + reason: flagged + ? `Found ${profaneWordList.length} potential profanity matches` + : 'No profanity detected', }; } private checkProfanityWithContextAware(text: string): CheckProfanityResult { const variants = this.getTextVariants(text, true); + const containsProfanity = this.isProfaneWithContextAware(text); const profaneWords: string[] = []; const severityMap: Record = {}; const matches: Match[] = []; const seen = new Set(); if (this.dictionaryMatcher) { - this.collectContextAwareCandidatesFromAc( + this.collectContextAwareCandidatesFromAcWithFallback( text, variants, profaneWords, @@ -1174,7 +1265,13 @@ class Filter { this.debugLog('Detected:', profaneWords); } - return this.buildContextAwareResult(text, profaneWords, severityMap, matches); + return this.buildContextAwareResult( + text, + profaneWords, + severityMap, + matches, + containsProfanity, + ); } private getReplacementRegex(word: string): RegExp { @@ -1202,9 +1299,10 @@ class Filter { text: string, profaneWords: Set, severityMap: Record, + containsProfanity?: boolean, ): CheckProfanityResult { const profaneWordList = this.dedupeNestedProfaneWords(Array.from(profaneWords)).sort( - (a, b) => b.length - a.length, + (a, b) => b.length - a.length || a.localeCompare(b), ); let processedText = text; @@ -1217,14 +1315,22 @@ class Filter { } } + const flagged = + containsProfanity !== undefined + ? containsProfanity + : profaneWordList.length > 0; + return { - containsProfanity: profaneWordList.length > 0, + containsProfanity: flagged, profaneWords: profaneWordList, processedText: this.replaceWith ? processedText : undefined, severityMap: this.severityLevels && Object.keys(severityMap).length > 0 ? severityMap : undefined, + reason: flagged + ? `Found ${profaneWordList.length} potential profanity matches` + : 'No profanity detected', }; } diff --git a/packages/js/src/utils/leetspeak.ts b/packages/js/src/utils/leetspeak.ts index aa2e1b6..687049b 100644 --- a/packages/js/src/utils/leetspeak.ts +++ b/packages/js/src/utils/leetspeak.ts @@ -176,6 +176,8 @@ const AGGRESSIVE_VOWEL_SUBSTITUTIONS: Record = { '@': 'u', }; +export { MODERATE_SUBSTITUTIONS, AGGRESSIVE_SUBSTITUTIONS }; + function applyCharSubstitutionsWithMap( text: string, substitutions: Record, diff --git a/packages/js/src/utils/variantMapping.ts b/packages/js/src/utils/variantMapping.ts index 7a0d7d8..d4e6892 100644 --- a/packages/js/src/utils/variantMapping.ts +++ b/packages/js/src/utils/variantMapping.ts @@ -3,6 +3,7 @@ * Assumes normalization is order-preserving (no reordering of characters). */ +import { AGGRESSIVE_SUBSTITUTIONS, MODERATE_SUBSTITUTIONS } from './leetspeak'; import { homoglyphToAscii } from './unicode'; export interface OriginalSpan { @@ -25,8 +26,28 @@ const LEET_TO_ASCII: Record = { '9': 'g', }; +const LEET_SUBSTITUTIONS: Record = { + ...MODERATE_SUBSTITUTIONS, + ...AGGRESSIVE_SUBSTITUTIONS, +}; + const SKIPPABLE_ORIGINAL_CHARS = new Set(['*', '.', '_', '-', ' ']); +/** Keep leetspeak/masking symbols when they belong to the obfuscated token. */ +const EDGE_MASKING_CHARS = new Set(['@', '$', '!', '#', '*']); + +function normalizeCharForAlign(char: string): string { + const mapped = homoglyphToAscii(char); + const decomposed = mapped.normalize('NFKD'); + let base = ''; + for (const codePoint of decomposed) { + if (!/\p{M}/u.test(codePoint)) { + base += codePoint; + } + } + return base.toLowerCase(); +} + function charsEqual(a: string, b: string): boolean { return a === b || a.toLowerCase() === b.toLowerCase(); } @@ -36,13 +57,25 @@ function charsAlign(originalChar: string, variantChar: string): boolean { return true; } + const originalNorm = normalizeCharForAlign(originalChar); + const variantNorm = normalizeCharForAlign(variantChar); + if (originalNorm && originalNorm === variantNorm) { + return true; + } + const leet = LEET_TO_ASCII[originalChar] ?? LEET_TO_ASCII[originalChar.toLowerCase()]; - if (leet !== undefined && charsEqual(leet, variantChar)) { + if (leet !== undefined && leet.toLowerCase() === variantNorm) { + return true; + } + + const substituted = + LEET_SUBSTITUTIONS[originalChar] ?? LEET_SUBSTITUTIONS[originalChar.toLowerCase()]; + if (substituted !== undefined && substituted.toLowerCase() === variantNorm) { return true; } const homoglyph = homoglyphToAscii(originalChar); - if (homoglyph !== originalChar && charsEqual(homoglyph, variantChar)) { + if (homoglyph !== originalChar && normalizeCharForAlign(homoglyph) === variantNorm) { return true; } @@ -53,6 +86,54 @@ function isSkippableOriginalChar(char: string): boolean { return SKIPPABLE_ORIGINAL_CHARS.has(char); } +function isCombiningMark(char: string): boolean { + return /\p{M}/u.test(char); +} + +function shouldTrimEdgePunctuation(char: string): boolean { + if (EDGE_MASKING_CHARS.has(char)) { + return false; + } + if ('._-'.includes(char)) { + return true; + } + return /\p{P}|\p{Z}/u.test(char); +} + +/** Drop leading/trailing whitespace and outer punctuation from a profane span. */ +export function trimProfaneSpanEdges(text: string, start: number, end: number): OriginalSpan { + if (start >= end) { + return { start, end, matchedText: '' }; + } + + while (start < end && /\s/u.test(text[start]!)) { + start++; + } + while (start < end && /\s/u.test(text[end - 1]!)) { + end--; + } + + while (start < end && '._-'.includes(text[start]!)) { + start++; + } + while (start < end && '._-'.includes(text[end - 1]!)) { + end--; + } + + while (start < end && shouldTrimEdgePunctuation(text[start]!)) { + start++; + } + while (start < end && shouldTrimEdgePunctuation(text[end - 1]!)) { + end--; + } + + return { start, end, matchedText: text.slice(start, end) }; +} + +function finalizeSpan(original: string, start: number, end: number): OriginalSpan { + return trimProfaneSpanEdges(original, start, end); +} + function fallbackSpan( original: string, variant: string, @@ -68,21 +149,14 @@ function fallbackSpan( const lowerNeedle = needle.toLowerCase(); const idx = lowerOriginal.indexOf(lowerNeedle); if (idx >= 0) { - return { - start: idx, - end: idx + needle.length, - matchedText: original.slice(idx, idx + needle.length), - }; - } - - return { - start: Math.min(variantStart, original.length), - end: Math.min(variantEnd, original.length), - matchedText: original.slice( - Math.min(variantStart, original.length), - Math.min(variantEnd, original.length), - ), - }; + return finalizeSpan(original, idx, idx + needle.length); + } + + return finalizeSpan( + original, + Math.min(variantStart, original.length), + Math.min(variantEnd, original.length), + ); } /** @@ -100,11 +174,7 @@ export function mapVariantSpanToOriginal( } if (original === variant) { - return { - start: variantStart, - end: variantEnd, - matchedText: original.slice(variantStart, variantEnd), - }; + return finalizeSpan(original, variantStart, variantEnd); } let originalIndex = 0; @@ -129,6 +199,11 @@ export function mapVariantSpanToOriginal( const variantChar = variant[variantIndex]!; const originalChar = original[originalIndex]!; + if (isCombiningMark(originalChar)) { + originalIndex++; + continue; + } + if (charsAlign(originalChar, variantChar)) { variantIndex++; originalIndex++; @@ -143,19 +218,66 @@ export function mapVariantSpanToOriginal( originalIndex++; } - if (origStart === -1) { - origStart = 0; + if (origEnd === -1 && variantIndex >= variantEnd) { + origEnd = originalIndex; + } + + if (origStart === -1 || origEnd === -1) { + return fallbackSpan(original, variant, variantStart, variantEnd); } - if (origEnd === -1) { - origEnd = original.length; + + if (origStart >= origEnd) { + return fallbackSpan(original, variant, variantStart, variantEnd); } const matchedText = original.slice(origStart, origEnd); - if (!matchedText || origStart >= origEnd) { + if (!matchedText) { return fallbackSpan(original, variant, variantStart, variantEnd); } - return { start: origStart, end: origEnd, matchedText }; + return finalizeSpan(original, origStart, origEnd); +} + +export interface ProfaneSpan { + word: string; + start: number; + end: number; +} + +/** Keep longest original-text span when starts or ranges overlap. */ +export function dedupeProfaneSpansByOverlap( + spans: Array<[string, number, number]>, +): Array<[string, number, number]> { + if (spans.length === 0) { + return []; + } + + const ordered = [...spans].sort( + (left, right) => left[1] - right[1] || right[2] - right[1] - (left[2] - left[1]), + ); + let kept: Array<[string, number, number]> = []; + + for (const candidate of ordered) { + const [word, start, end] = candidate; + const length = end - start; + + if ( + kept.some( + ([, keptStart, keptEnd]) => + start >= keptStart && end <= keptEnd && keptEnd - keptStart > length, + ) + ) { + continue; + } + + kept = kept.filter( + ([, keptStart, keptEnd]) => + !(keptStart >= start && keptEnd <= end && length > keptEnd - keptStart), + ); + kept.push([word, start, end]); + } + + return kept; } /** True when `shorter` is a word-bounded substring of `longer` (avoids ass/classic). */ diff --git a/packages/js/tests/context-aware.test.ts b/packages/js/tests/context-aware.test.ts index ba077da..69b7c5d 100644 --- a/packages/js/tests/context-aware.test.ts +++ b/packages/js/tests/context-aware.test.ts @@ -208,4 +208,35 @@ describe('Context-Aware Filtering', () => { expect(result.reason).toBeDefined(); }); }); + + describe('Result shape parity', () => { + it('includes reason on AC path', () => { + const acFilter = new Filter({ + languages: ['english'], + detectLeetspeak: true, + }); + const result = acFilter.checkProfanity('fuuuuuck'); + expect(result.reason).toBe('Found 1 potential profanity matches'); + }); + + it('aligns containsProfanity with isProfane for whitelisted context', () => { + const result = filter.checkProfanity('This movie is the bomb'); + expect(result.containsProfanity).toBe(false); + expect(result.reason).toBe('No profanity detected'); + expect(filter.isProfane('This movie is the bomb')).toBe(result.containsProfanity); + }); + + it('aligns containsProfanity with isProfane for repeated chars', () => { + const leetFilter = new Filter({ + languages: ['english'], + detectLeetspeak: true, + enableContextAware: true, + }); + const result = leetFilter.checkProfanity('fuuuuuck'); + expect(result.containsProfanity).toBe(true); + expect(result.profaneWords).toEqual(['fuuuuuck']); + expect(result.reason).toBe('Found 1 potential profanity matches'); + expect(leetFilter.isProfane('fuuuuuck')).toBe(result.containsProfanity); + }); + }); }); \ No newline at end of file diff --git a/packages/js/tests/variant-mapping.test.ts b/packages/js/tests/variant-mapping.test.ts index 504713d..b713cb2 100644 --- a/packages/js/tests/variant-mapping.test.ts +++ b/packages/js/tests/variant-mapping.test.ts @@ -1,4 +1,4 @@ -import { mapVariantSpanToOriginal, isNestedProfaneSpan } from '../src/utils/variantMapping'; +import { mapVariantSpanToOriginal, isNestedProfaneSpan, trimProfaneSpanEdges } from '../src/utils/variantMapping'; describe('mapVariantSpanToOriginal', () => { test('maps collapsed separators back to original span', () => { @@ -45,6 +45,25 @@ describe('mapVariantSpanToOriginal', () => { expect(span.start).toBe(6); expect(span.end).toBe(10); }); + + test('maps accent-stripped variant back to original word', () => { + const original = ' mamá se fue?'; + const variant = ' mama se fue?'; + const span = mapVariantSpanToOriginal(original, variant, 1, 5); + expect(span.matchedText).toBe('mamá'); + }); + + test('maps leetspeak parenthesis substitution back to original span', () => { + const original = '(miro su camel toe bien marcado en sus tangas)'; + const variant = 'cmiro su camel toe bien marcado en sus tangas)'; + const span = mapVariantSpanToOriginal(original, variant, 9, 18); + expect(span.matchedText).toBe('camel toe'); + }); + + test('trims leading dots from gay span edges', () => { + const span = trimProfaneSpanEdges('Sei un...gay?', 7, 12); + expect(span.matchedText).toBe('gay'); + }); }); describe('isNestedProfaneSpan', () => { diff --git a/packages/py/glin_profanity/filters/filter.py b/packages/py/glin_profanity/filters/filter.py index 6b7b4b2..f804980 100644 --- a/packages/py/glin_profanity/filters/filter.py +++ b/packages/py/glin_profanity/filters/filter.py @@ -21,6 +21,8 @@ ) from glin_profanity.utils.evasion import normalize_evasion from glin_profanity.utils.variant_mapping import ( + ProfaneSpan, + dedupe_profane_spans_by_overlap, is_nested_profane_span, map_variant_span_to_original, ) @@ -237,9 +239,6 @@ def _resolve_ac_match_in_original( match: DictionaryMatch, is_original_variant: bool, ) -> tuple[str, int, int]: - if is_original_variant and variant_text == original_text: - return match.matched_text, match.start, match.end - span = map_variant_span_to_original( original_text, variant_text, @@ -257,9 +256,6 @@ def _resolve_regex_match_in_original( matched_word: str, is_original_variant: bool, ) -> tuple[str, int, int]: - if is_original_variant and variant_text == original_text: - return matched_word, match_start, match_end - span = map_variant_span_to_original( original_text, variant_text, @@ -288,13 +284,26 @@ def _evaluate_severity_on_variants( return None + def _profane_words_from_spans( + self, + spans: list[ProfaneSpan], + severity_map: dict[str, SeverityLevel] | None = None, + ) -> tuple[set[str], dict[str, SeverityLevel]]: + deduped = dedupe_profane_spans_by_overlap(spans) + source_severity = severity_map or {} + words = {word for word, _start, _end in deduped if word} + resolved_severity = { + word: source_severity.get(word, SeverityLevel.EXACT) for word in words + } + return words, resolved_severity + def _collect_matches_from_variant( self, dict_word: str, variant_text: str, original_text: str, severity: SeverityLevel, - profane_words: set[str], + profane_spans: list[ProfaneSpan], severity_map: dict[str, SeverityLevel], matches: list[Match], is_original_variant: bool, @@ -310,15 +319,20 @@ def _collect_matches_from_variant( ): continue - matched_word, resolved_start, _ = self._resolve_regex_match_in_original( - original_text, - variant_text, - start, - end, - match.group(0) if is_original_variant else dict_word, - is_original_variant, + matched_word, resolved_start, resolved_end = ( + self._resolve_regex_match_in_original( + original_text, + variant_text, + start, + end, + match.group(0) if is_original_variant else dict_word, + is_original_variant, + ) ) - profane_words.add(matched_word) + if not matched_word: + continue + + profane_spans.append((matched_word, resolved_start, resolved_end)) if matched_word not in severity_map: severity_map[matched_word] = severity @@ -330,6 +344,59 @@ def _collect_matches_from_variant( } ) + def _collect_profane_spans_from_variant_ac( + self, + text: str, + variant_text: str, + ) -> list[ProfaneSpan]: + matcher = self.dictionary_matcher + if matcher is None: + return [] + + options = self._get_dictionary_search_options() + spans: list[ProfaneSpan] = [] + + for match in matcher.find_matches(variant_text, options): + matched_word, start, end = self._resolve_ac_match_in_original( + text, + variant_text, + match, + False, + ) + if matched_word: + spans.append((matched_word, start, end)) + + return spans + + def _collect_profane_spans_ac( + self, + text: str, + variants: dict[str, str], + contains_profanity: bool = False, + ) -> list[ProfaneSpan]: + spans = self._collect_profane_spans_from_variant_ac( + text, variants["normalized"] + ) + if spans or not contains_profanity: + return dedupe_profane_spans_by_overlap(spans) + + if variants["original"] != variants["normalized"]: + spans = self._collect_profane_spans_from_variant_ac( + text, variants["original"] + ) + if spans: + return dedupe_profane_spans_by_overlap(spans) + + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + spans = self._collect_profane_spans_from_variant_ac( + text, variants["aggressive"] + ) + + return dedupe_profane_spans_by_overlap(spans) + def _normalize_obfuscated(self, text: str) -> str: """ Normalize obfuscated text by replacing common character substitutions. @@ -547,89 +614,112 @@ def _is_profane_legacy(self, value: str) -> bool: def _check_profanity_with_aho_corasick(self, text: str) -> CheckProfanityResult: variants = self._get_text_variants(text, True) - profane_words: set[str] = set() severity_map: dict[str, SeverityLevel] = {} - options = self._get_dictionary_search_options() - matcher = self.dictionary_matcher - assert matcher is not None + contains_profanity = self._is_profane_with_aho_corasick(text) + profane_spans = self._collect_profane_spans_ac( + text, variants, contains_profanity=contains_profanity + ) + profane_words, severity_map = self._profane_words_from_spans( + profane_spans, severity_map + ) + return self._build_profanity_result( + text, + profane_words, + severity_map, + contains_profanity=contains_profanity, + ) - def collect_from_variant(variant_text: str, is_original: bool) -> bool: - for match in matcher.find_matches(variant_text, options): - matched_word, _, _ = self._resolve_ac_match_in_original( - text, variant_text, match, is_original - ) - if not matched_word: - continue - profane_words.add(matched_word) - if matched_word not in severity_map: - severity_map[matched_word] = SeverityLevel.EXACT - return False + def _collect_legacy_spans_from_variant( + self, + text: str, + variants: dict[str, str], + variant_key: str, + is_original_variant: bool, + profane_spans: list[ProfaneSpan], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + ) -> None: + variant_text = variants[variant_key] + for dict_word in self.words: + if dict_word.lower() in self.ignore_words: + continue - self._for_each_text_variant(variants, collect_from_variant) - return self._build_profanity_result(text, profane_words, severity_map) + severity = self._evaluate_severity(dict_word, variant_text) + if severity is not None: + self._collect_matches_from_variant( + dict_word, + variant_text, + text, + severity, + profane_spans, + severity_map, + matches, + is_original_variant, + ) def _check_profanity_legacy_non_context(self, text: str) -> CheckProfanityResult: variants = self._get_text_variants(text, True) - profane_words_set: set[str] = set() + profane_spans: list[ProfaneSpan] = [] severity_map: dict[str, SeverityLevel] = {} matches: list[Match] = [] + contains_profanity = False for dict_word in self.words: if dict_word.lower() in self.ignore_words: continue - severity = self._evaluate_severity(dict_word, variants["original"]) - if severity is not None: - self._collect_matches_from_variant( - dict_word, - variants["original"], + if self._evaluate_severity_on_variants(dict_word, variants) is not None: + contains_profanity = True + + self._collect_legacy_spans_from_variant( + text, + variants, + "normalized", + False, + profane_spans, + severity_map, + matches, + ) + if not profane_spans and contains_profanity: + if variants["original"] != variants["normalized"]: + self._collect_legacy_spans_from_variant( text, - severity, - profane_words_set, + variants, + "original", + True, + profane_spans, severity_map, matches, - True, ) - - if variants["normalized"] != variants["original"]: - severity = self._evaluate_severity(dict_word, variants["normalized"]) - if severity is not None: - self._collect_matches_from_variant( - dict_word, - variants["normalized"], - text, - severity, - profane_words_set, - severity_map, - matches, - False, - ) - if ( - variants["aggressive"] != variants["normalized"] + not profane_spans + and variants["aggressive"] != variants["normalized"] and variants["aggressive"] != variants["original"] ): - severity = self._evaluate_severity(dict_word, variants["aggressive"]) - if severity is not None: - self._collect_matches_from_variant( - dict_word, - variants["aggressive"], - text, - severity, - profane_words_set, - severity_map, - matches, - False, - ) - - result = self._build_profanity_result(text, profane_words_set, severity_map) - if matches: - result["matches"] = matches - result["reason"] = ( - f"Found {len(matches)} potential profanity matches" - if matches - else "No profanity detected" + self._collect_legacy_spans_from_variant( + text, + variants, + "aggressive", + False, + profane_spans, + severity_map, + matches, + ) + + profane_words, severity_map = self._profane_words_from_spans( + profane_spans, severity_map ) + result = self._build_profanity_result( + text, + profane_words, + severity_map, + contains_profanity=contains_profanity, + ) + if matches: + deduped_words = set(result["profane_words"]) + result["matches"] = [ + match for match in matches if match["word"] in deduped_words + ] return result def _dedupe_nested_profane_words(self, words: list[str]) -> list[str]: @@ -647,11 +737,11 @@ def _build_profanity_result( text: str, profane_words: set[str], severity_map: dict[str, SeverityLevel], + contains_profanity: bool | None = None, ) -> CheckProfanityResult: profane_word_list = sorted( self._dedupe_nested_profane_words(list(profane_words)), - key=len, - reverse=True, + key=lambda word: (-len(word), word), ) processed_text = text @@ -660,12 +750,17 @@ def _build_profanity_result( replacement_regex = self._get_replacement_regex(word) processed_text = replacement_regex.sub(self.replace_with, processed_text) + flagged = ( + contains_profanity + if contains_profanity is not None + else len(profane_word_list) > 0 + ) result: CheckProfanityResult = { - "contains_profanity": len(profane_word_list) > 0, + "contains_profanity": flagged, "profane_words": profane_word_list, "reason": ( f"Found {len(profane_word_list)} potential profanity matches" - if profane_word_list + if flagged else "No profanity detected" ), } @@ -739,16 +834,19 @@ def _collect_context_aware_candidates_from_ac( severity_map: dict[str, SeverityLevel], matches: list[Match], seen: set[str], + variant_key: str = "normalized", ) -> None: options = self._get_dictionary_search_options() matcher = self.dictionary_matcher assert matcher is not None + variant_text = variants[variant_key] - def process_ac_match( - variant_text: str, match: DictionaryMatch, is_original: bool - ) -> None: + for match in matcher.find_matches(variant_text, options): matched_word, start, _ = self._resolve_ac_match_in_original( - text, variant_text, match, is_original + text, + variant_text, + match, + False, ) self._record_context_aware_match( text, @@ -761,12 +859,47 @@ def process_ac_match( seen, ) - def collect_from_variant(variant_text: str, is_original: bool) -> bool: - for match in matcher.find_matches(variant_text, options): - process_ac_match(variant_text, match, is_original) - return False + def _collect_context_aware_candidates_from_ac_with_fallback( + self, + text: str, + variants: dict[str, str], + profane_words: list[str], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + seen: set[str], + ) -> None: + self._collect_context_aware_candidates_from_ac( + text, variants, profane_words, severity_map, matches, seen + ) + if profane_words: + return - self._for_each_text_variant(variants, collect_from_variant) + if variants["original"] != variants["normalized"]: + self._collect_context_aware_candidates_from_ac( + text, + variants, + profane_words, + severity_map, + matches, + seen, + variant_key="original", + ) + if profane_words: + return + + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + self._collect_context_aware_candidates_from_ac( + text, + variants, + profane_words, + severity_map, + matches, + seen, + variant_key="aggressive", + ) def _collect_context_aware_candidates_from_legacy_fuzzy( self, @@ -781,28 +914,18 @@ def _collect_context_aware_candidates_from_legacy_fuzzy( if dict_word.lower() in self.ignore_words: continue - def collect_from_variant(variant_text: str) -> None: - if self._evaluate_severity(dict_word, variant_text) != SeverityLevel.FUZZY: - return - self._record_context_aware_match( - text, - dict_word, - 0, - SeverityLevel.FUZZY, - profane_words, - severity_map, - matches, - seen, - ) - - collect_from_variant(variants["original"]) - if variants["normalized"] != variants["original"]: - collect_from_variant(variants["normalized"]) - if ( - variants["aggressive"] != variants["normalized"] - and variants["aggressive"] != variants["original"] - ): - collect_from_variant(variants["aggressive"]) + if self._evaluate_severity(dict_word, variants["normalized"]) != SeverityLevel.FUZZY: + continue + self._record_context_aware_match( + text, + dict_word, + 0, + SeverityLevel.FUZZY, + profane_words, + severity_map, + matches, + seen, + ) def _collect_context_aware_candidates_from_legacy( self, @@ -817,62 +940,51 @@ def _collect_context_aware_candidates_from_legacy( if dict_word.lower() in self.ignore_words: continue - def collect_from_variant(variant_text: str, is_original: bool) -> None: - severity = self._evaluate_severity(dict_word, variant_text) - if severity is None: - return - - if not self.word_boundaries and severity == SeverityLevel.FUZZY: - self._record_context_aware_match( - text, - dict_word, - 0, - severity, - profane_words, - severity_map, - matches, - seen, - ) - return - - regex = self._get_regex(dict_word) - script = self._get_word_script(dict_word) - for match in regex.finditer(variant_text): - start = match.start() - end = match.end() - if not match_has_word_boundary( - variant_text, start, end, script, self.word_boundaries - ): - continue - matched_word, resolved_start, _ = self._resolve_regex_match_in_original( - text, - variant_text, - start, - end, - match.group(0) if is_original else dict_word, - is_original, - ) - self._record_context_aware_match( - text, - matched_word, - resolved_start, - severity, - profane_words, - severity_map, - matches, - seen, - ) - - collect_from_variant(variants["original"], True) - - if variants["normalized"] != variants["original"]: - collect_from_variant(variants["normalized"], False) + severity = self._evaluate_severity(dict_word, variants["normalized"]) + if severity is None: + continue - if ( - variants["aggressive"] != variants["normalized"] - and variants["aggressive"] != variants["original"] - ): - collect_from_variant(variants["aggressive"], False) + if not self.word_boundaries and severity == SeverityLevel.FUZZY: + self._record_context_aware_match( + text, + dict_word, + 0, + severity, + profane_words, + severity_map, + matches, + seen, + ) + continue + + regex = self._get_regex(dict_word) + script = self._get_word_script(dict_word) + variant_text = variants["normalized"] + for match in regex.finditer(variant_text): + start = match.start() + end = match.end() + if not match_has_word_boundary( + variant_text, start, end, script, self.word_boundaries + ): + continue + matched_word, resolved_start, _ = self._resolve_regex_match_in_original( + text, + variant_text, + start, + end, + dict_word, + False, + ) + self._record_context_aware_match( + text, + matched_word, + resolved_start, + severity, + profane_words, + severity_map, + matches, + seen, + ) def _build_context_aware_result( self, @@ -880,14 +992,15 @@ def _build_context_aware_result( profane_words: list[str], severity_map: dict[str, SeverityLevel], matches: list[Match], + contains_profanity: bool | None = None, ) -> CheckProfanityResult: + profane_word_list = sorted( + self._dedupe_nested_profane_words(list(dict.fromkeys(profane_words))), + key=lambda word: (-len(word), word), + ) processed_text = text - if self.replace_with and profane_words: - for word in sorted( - self._dedupe_nested_profane_words(list(dict.fromkeys(profane_words))), - key=len, - reverse=True, - ): + if self.replace_with and profane_word_list: + for word in profane_word_list: processed_text = self._get_replacement_regex(word).sub( self.replace_with, processed_text ) @@ -898,19 +1011,22 @@ def _build_context_aware_result( match.get("context_score") or 0.5 for match in matches ) / len(matches) + flagged = ( + contains_profanity + if contains_profanity is not None + else len(profane_word_list) > 0 + ) result: CheckProfanityResult = { - "contains_profanity": len(profane_words) > 0, - "profane_words": sorted( - self._dedupe_nested_profane_words(list(dict.fromkeys(profane_words))) - ), + "contains_profanity": flagged, + "profane_words": profane_word_list, "reason": ( - f"Found {len(matches)} potential profanity matches" - if matches + f"Found {len(profane_word_list)} potential profanity matches" + if flagged else "No profanity detected" ), } - if self.replace_with: + if self.replace_with and profane_word_list: result["processed_text"] = processed_text if self.severity_levels and severity_map: result["severity_map"] = severity_map @@ -925,13 +1041,14 @@ def _build_context_aware_result( def _check_profanity_with_context_aware(self, text: str) -> CheckProfanityResult: variants = self._get_text_variants(text, True) + contains_profanity = self._is_profane_with_context_aware(text) profane_words: list[str] = [] severity_map: dict[str, SeverityLevel] = {} matches: list[Match] = [] seen: set[str] = set() if self.dictionary_matcher: - self._collect_context_aware_candidates_from_ac( + self._collect_context_aware_candidates_from_ac_with_fallback( text, variants, profane_words, severity_map, matches, seen ) if not self.word_boundaries: @@ -947,7 +1064,11 @@ def _check_profanity_with_context_aware(self, text: str) -> CheckProfanityResult self._debug_log("Detected:", profane_words) return self._build_context_aware_result( - text, profane_words, severity_map, matches + text, + profane_words, + severity_map, + matches, + contains_profanity=contains_profanity, ) def _has_context_aware_ac_match(self, text: str, variants: dict[str, str]) -> bool: diff --git a/packages/py/glin_profanity/utils/variant_mapping.py b/packages/py/glin_profanity/utils/variant_mapping.py index 85b6376..a865a33 100644 --- a/packages/py/glin_profanity/utils/variant_mapping.py +++ b/packages/py/glin_profanity/utils/variant_mapping.py @@ -3,8 +3,10 @@ from __future__ import annotations import re +import unicodedata from dataclasses import dataclass +from glin_profanity.utils.leetspeak import AGGRESSIVE_SUBSTITUTIONS, MODERATE_SUBSTITUTIONS from glin_profanity.utils.unicode import homoglyph_to_ascii _LEET_TO_ASCII = { @@ -21,8 +23,16 @@ "9": "g", } +_LEET_SUBSTITUTIONS: dict[str, str] = { + **MODERATE_SUBSTITUTIONS, + **AGGRESSIVE_SUBSTITUTIONS, +} + _SKIPPABLE_ORIGINAL_CHARS = {"*", ".", "_", "-", " "} +# Keep leetspeak/masking symbols when they belong to the obfuscated token. +_EDGE_MASKING_CHARS = frozenset("@$!#*") + @dataclass(frozen=True) class OriginalSpan: @@ -31,6 +41,13 @@ class OriginalSpan: matched_text: str +def _normalize_char_for_align(char: str) -> str: + mapped = homoglyph_to_ascii(char) + decomposed = unicodedata.normalize("NFKD", mapped) + base = "".join(c for c in decomposed if unicodedata.combining(c) == 0) + return base.lower() + + def _chars_equal(left: str, right: str) -> bool: return left == right or left.lower() == right.lower() @@ -39,14 +56,25 @@ def _chars_align(original_char: str, variant_char: str) -> bool: if _chars_equal(original_char, variant_char): return True + original_norm = _normalize_char_for_align(original_char) + variant_norm = _normalize_char_for_align(variant_char) + if original_norm and original_norm == variant_norm: + return True + mapped = _LEET_TO_ASCII.get(original_char) or _LEET_TO_ASCII.get( original_char.lower() ) - if mapped is not None and _chars_equal(mapped, variant_char): + if mapped is not None and mapped.lower() == variant_norm: + return True + + substituted = _LEET_SUBSTITUTIONS.get(original_char) or _LEET_SUBSTITUTIONS.get( + original_char.lower() + ) + if substituted is not None and substituted.lower() == variant_norm: return True homoglyph = homoglyph_to_ascii(original_char) - if homoglyph != original_char and _chars_equal(homoglyph, variant_char): + if homoglyph != original_char and _normalize_char_for_align(homoglyph) == variant_norm: return True return False @@ -56,6 +84,46 @@ def _is_skippable_original_char(char: str) -> bool: return char in _SKIPPABLE_ORIGINAL_CHARS +def _is_combining_mark(char: str) -> bool: + return bool(char) and unicodedata.combining(char) != 0 + + +def _should_trim_edge_punctuation(char: str) -> bool: + if char in _EDGE_MASKING_CHARS: + return False + if char in {".", "_", "-"}: + return True + category = unicodedata.category(char) + return category.startswith("P") or category.startswith("Z") + + +def trim_profane_span_edges(text: str, start: int, end: int) -> OriginalSpan: + """Drop leading/trailing whitespace and outer punctuation from a profane span.""" + if start >= end: + return OriginalSpan(start=start, end=end, matched_text="") + + while start < end and text[start].isspace(): + start += 1 + while start < end and text[end - 1].isspace(): + end -= 1 + + while start < end and text[start] in {".", "_", "-"}: + start += 1 + while start < end and text[end - 1] in {".", "_", "-"}: + end -= 1 + + while start < end and _should_trim_edge_punctuation(text[start]): + start += 1 + while start < end and _should_trim_edge_punctuation(text[end - 1]): + end -= 1 + + return OriginalSpan(start=start, end=end, matched_text=text[start:end]) + + +def _finalize_span(original: str, start: int, end: int) -> OriginalSpan: + return trim_profane_span_edges(original, start, end) + + def _fallback_span( original: str, variant: str, variant_start: int, variant_end: int ) -> OriginalSpan: @@ -65,15 +133,11 @@ def _fallback_span( idx = original.lower().find(needle.lower()) if idx >= 0: - return OriginalSpan( - start=idx, - end=idx + len(needle), - matched_text=original[idx : idx + len(needle)], - ) + return _finalize_span(original, idx, idx + len(needle)) start = min(variant_start, len(original)) end = min(variant_end, len(original)) - return OriginalSpan(start=start, end=end, matched_text=original[start:end]) + return _finalize_span(original, start, end) def map_variant_span_to_original( @@ -87,11 +151,7 @@ def map_variant_span_to_original( return OriginalSpan(start=0, end=0, matched_text="") if original == variant: - return OriginalSpan( - start=variant_start, - end=variant_end, - matched_text=original[variant_start:variant_end], - ) + return _finalize_span(original, variant_start, variant_end) original_index = 0 variant_index = 0 @@ -112,6 +172,10 @@ def map_variant_span_to_original( variant_char = variant[variant_index] original_char = original[original_index] + if _is_combining_mark(original_char): + original_index += 1 + continue + if _chars_align(original_char, variant_char): variant_index += 1 original_index += 1 @@ -123,20 +187,57 @@ def map_variant_span_to_original( original_index += 1 - if orig_start == -1: - orig_start = 0 - if orig_end == -1: - orig_end = len(original) + if orig_end == -1 and variant_index >= variant_end: + orig_end = original_index + + if orig_start == -1 or orig_end == -1: + return _fallback_span(original, variant, variant_start, variant_end) + + if orig_start >= orig_end: + return _fallback_span(original, variant, variant_start, variant_end) matched_text = original[orig_start:orig_end] - if not matched_text or orig_start >= orig_end: + if not matched_text: return _fallback_span(original, variant, variant_start, variant_end) - return OriginalSpan( - start=orig_start, - end=orig_end, - matched_text=matched_text, - ) + return _finalize_span(original, orig_start, orig_end) + + +ProfaneSpan = tuple[str, int, int] + + +def dedupe_profane_spans_by_overlap(spans: list[ProfaneSpan]) -> list[ProfaneSpan]: + """Keep longest original-text span when starts or ranges overlap.""" + if not spans: + return [] + + ordered = sorted(spans, key=lambda item: (item[1], -(item[2] - item[1]))) + kept: list[ProfaneSpan] = [] + + for candidate in ordered: + word, start, end = candidate + length = end - start + + if any( + start >= kept_start + and end <= kept_end + and (kept_end - kept_start) > length + for _, kept_start, kept_end in kept + ): + continue + + kept = [ + item + for item in kept + if not ( + item[1] >= start + and item[2] <= end + and length > (item[2] - item[1]) + ) + ] + kept.append((word, start, end)) + + return kept _NESTED_WORD_BOUNDARY = re.compile(r"\w") diff --git a/packages/py/tests/test_profane_words_collection.py b/packages/py/tests/test_profane_words_collection.py new file mode 100644 index 0000000..7e02f31 --- /dev/null +++ b/packages/py/tests/test_profane_words_collection.py @@ -0,0 +1,110 @@ +"""Tests for profane_words collection from normalized tier with fallbacks.""" + +from glin_profanity import Filter +from glin_profanity.utils.variant_mapping import dedupe_profane_spans_by_overlap + + +class TestProfaneSpanDedupe: + def test_keeps_longer_same_start_prefix(self) -> None: + spans = [("Am", 0, 2), ("Amm", 0, 3)] + assert dedupe_profane_spans_by_overlap(spans) == [("Amm", 0, 3)] + + +class TestNormalizedProfaneWordsCollection: + @classmethod + def setup_class(cls) -> None: + cls.filter = Filter( + { + "all_languages": True, + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) + cls.english_filter = Filter( + { + "languages": ["english"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) + + def test_amm_bno_q_haces(self) -> None: + result = self.filter.check_profanity("Amm bno q haces") + assert result["contains_profanity"] is True + assert result["profane_words"] == ["Amm"] + + def test_cuéntame_only_reports_con(self) -> None: + text = "Cuéntame algo *Digo con voz adormilada* Me gusta tu voz" + result = self.filter.check_profanity(text) + assert "Cu" not in result["profane_words"] + assert "con" in result["profane_words"] + + def test_f_dot_u_c_k_maps_to_original(self) -> None: + result = self.filter.check_profanity("say f.u.c.k off") + assert result["profane_words"] == ["f.u.c.k"] + + def test_cjk_still_maps_to_original(self) -> None: + result = self.filter.check_profanity("123肏456") + assert result["contains_profanity"] is True + assert result["profane_words"] == ["肏"] + + def test_f_at_ck_fallback_to_original(self) -> None: + result = self.english_filter.check_profanity("f@ck") + assert result["contains_profanity"] is True + assert result["profane_words"] == ["f@ck"] + + def test_repeated_chars_fallback_to_original(self) -> None: + for text in ("fuuuuuck", "fffffffuck"): + result = self.english_filter.check_profanity(text) + assert result["contains_profanity"] is True + assert result["profane_words"] == [text] + + def test_contains_implies_non_empty_profane_words(self) -> None: + for text in ("f@ck", "fuuuuuck", "say f.u.c.k off", "123肏456"): + result = self.english_filter.check_profanity(text) + if result["contains_profanity"]: + assert len(result["profane_words"]) > 0 + + def test_replace_with_applies_when_contains_profanity(self) -> None: + masked = Filter( + { + "languages": ["english"], + "detect_leetspeak": True, + "replace_with": "***", + } + ).check_profanity("fuuuuuck") + assert masked["contains_profanity"] is True + assert masked["processed_text"] == "***" + + def test_context_aware_matches_is_profane_for_repeated_chars(self) -> None: + context_filter = Filter( + { + "languages": ["english"], + "detect_leetspeak": True, + "enable_context_aware": True, + } + ) + assert context_filter.is_profane("fuuuuuck") is True + result = context_filter.check_profanity("fuuuuuck") + assert result["contains_profanity"] is True + assert result["profane_words"] == ["fuuuuuck"] + assert result["reason"] == "Found 1 potential profanity matches" + + def test_ac_path_reason_matches_profane_word_count(self) -> None: + result = self.english_filter.check_profanity("fuuuuuck") + assert result["reason"] == "Found 1 potential profanity matches" + + def test_context_aware_whitelist_reason(self) -> None: + context_filter = Filter( + { + "languages": ["english"], + "enable_context_aware": True, + "context_window": 3, + "confidence_threshold": 0.7, + } + ) + result = context_filter.check_profanity("This movie is the bomb") + assert result["contains_profanity"] is False + assert result["profane_words"] == [] + assert result["reason"] == "No profanity detected" + assert context_filter.is_profane("This movie is the bomb") is False diff --git a/packages/py/tests/test_variant_mapping.py b/packages/py/tests/test_variant_mapping.py index e603d55..c3376cb 100644 --- a/packages/py/tests/test_variant_mapping.py +++ b/packages/py/tests/test_variant_mapping.py @@ -3,6 +3,7 @@ from glin_profanity.utils.variant_mapping import ( is_nested_profane_span, map_variant_span_to_original, + trim_profane_span_edges, ) @@ -45,6 +46,22 @@ def test_identity_when_texts_match(self) -> None: assert span.start == 6 assert span.end == 10 + def test_maps_accent_stripped_variant_to_original_word(self) -> None: + original = " mamá se fue?" + variant = " mama se fue?" + span = map_variant_span_to_original(original, variant, 1, 5) + assert span.matched_text == "mamá" + + def test_maps_leetspeak_parenthesis_substitution(self) -> None: + original = "(miro su camel toe bien marcado en sus tangas)" + variant = "cmiro su camel toe bien marcado en sus tangas)" + span = map_variant_span_to_original(original, variant, 9, 18) + assert span.matched_text == "camel toe" + + def test_trims_leading_dots_from_gay(self) -> None: + span = trim_profane_span_edges("Sei un...gay?", 7, 12) + assert span.matched_text == "gay" + class TestNestedProfaneSpan: def test_detects_nested_spans(self) -> None: From df6603c4c79fc9cf26645edfebfb82458ec733c0 Mon Sep 17 00:00:00 2001 From: wlike Date: Sat, 27 Jun 2026 18:55:26 +0800 Subject: [PATCH 04/11] fix: populate profane_words on legacy fuzzy and context-aware legacy paths Collect FUZZY matches when word boundaries are disabled, and add original/aggressive tier fallback for context-aware legacy collection so check_profanity stays aligned with is_profane. Co-authored-by: Cursor --- packages/js/src/filters/Filter.ts | 94 ++++++++++++++++--- packages/py/glin_profanity/filters/filter.py | 74 +++++++++++++-- .../py/tests/test_profane_words_collection.py | 32 +++++++ 3 files changed, 178 insertions(+), 22 deletions(-) diff --git a/packages/js/src/filters/Filter.ts b/packages/js/src/filters/Filter.ts index b569e24..7d10bbb 100644 --- a/packages/js/src/filters/Filter.ts +++ b/packages/js/src/filters/Filter.ts @@ -879,17 +879,27 @@ class Filter { if (this.ignoreWords.has(dictWord.toLowerCase())) continue; const severity = this.evaluateSeverity(dictWord, variantText); - if (severity !== undefined) { - this.collectMatchesFromVariant( - dictWord, - variantText, - text, - severity, - profaneSpans, - severityMap, - isOriginalVariant, - ); + if (severity === undefined) { + continue; } + + if (!this.wordBoundaries && severity === SeverityLevel.FUZZY) { + if (severityMap[dictWord] === undefined) { + profaneSpans.push([dictWord, 0, dictWord.length]); + severityMap[dictWord] = severity; + } + continue; + } + + this.collectMatchesFromVariant( + dictWord, + variantText, + text, + severity, + profaneSpans, + severityMap, + isOriginalVariant, + ); } } @@ -1112,13 +1122,17 @@ class Filter { severityMap: Record, matches: Match[], seen: Set, + variantKey: 'original' | 'normalized' | 'aggressive' = 'normalized', ): void { + const variantText = variants[variantKey]; + const isOriginal = variantKey === 'original'; + for (const dictWord of this.words.keys()) { if (this.ignoreWords.has(dictWord.toLowerCase())) { continue; } - const severity = this.evaluateSeverity(dictWord, variants.normalized); + const severity = this.evaluateSeverity(dictWord, variantText); if (severity === undefined) { continue; } @@ -1139,7 +1153,6 @@ class Filter { const regex = this.getRegex(dictWord); const script = this.getWordScript(dictWord); - const variantText = variants.normalized; let match: RegExpExecArray | null; while ((match = regex.exec(variantText)) !== null) { const start = match.index; @@ -1152,8 +1165,8 @@ class Filter { variantText, start, end, - dictWord, - false, + isOriginal ? match[0] : dictWord, + isOriginal, ); this.recordContextAwareMatch( text, @@ -1169,6 +1182,57 @@ class Filter { } } + private collectContextAwareCandidatesFromLegacyWithFallback( + text: string, + variants: { original: string; normalized: string; aggressive: string }, + profaneWords: string[], + severityMap: Record, + matches: Match[], + seen: Set, + ): void { + this.collectContextAwareCandidatesFromLegacy( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + ); + if (profaneWords.length > 0) { + return; + } + + if (variants.original !== variants.normalized) { + this.collectContextAwareCandidatesFromLegacy( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + 'original', + ); + } + if (profaneWords.length > 0) { + return; + } + + if ( + variants.aggressive !== variants.normalized && + variants.aggressive !== variants.original + ) { + this.collectContextAwareCandidatesFromLegacy( + text, + variants, + profaneWords, + severityMap, + matches, + seen, + 'aggressive', + ); + } + } + private buildContextAwareResult( text: string, profaneWords: string[], @@ -1251,7 +1315,7 @@ class Filter { ); } } else { - this.collectContextAwareCandidatesFromLegacy( + this.collectContextAwareCandidatesFromLegacyWithFallback( text, variants, profaneWords, diff --git a/packages/py/glin_profanity/filters/filter.py b/packages/py/glin_profanity/filters/filter.py index f804980..29e84a7 100644 --- a/packages/py/glin_profanity/filters/filter.py +++ b/packages/py/glin_profanity/filters/filter.py @@ -645,8 +645,23 @@ def _collect_legacy_spans_from_variant( continue severity = self._evaluate_severity(dict_word, variant_text) - if severity is not None: - self._collect_matches_from_variant( + if severity is None: + continue + + if not self.word_boundaries and severity == SeverityLevel.FUZZY: + if dict_word not in severity_map: + profane_spans.append((dict_word, 0, len(dict_word))) + severity_map[dict_word] = severity + matches.append( + { + "word": dict_word, + "index": 0, + "severity": severity, + } + ) + continue + + self._collect_matches_from_variant( dict_word, variant_text, text, @@ -935,12 +950,16 @@ def _collect_context_aware_candidates_from_legacy( severity_map: dict[str, SeverityLevel], matches: list[Match], seen: set[str], + variant_key: str = "normalized", ) -> None: + variant_text = variants[variant_key] + is_original = variant_key == "original" + for dict_word in self.words: if dict_word.lower() in self.ignore_words: continue - severity = self._evaluate_severity(dict_word, variants["normalized"]) + severity = self._evaluate_severity(dict_word, variant_text) if severity is None: continue @@ -959,7 +978,6 @@ def _collect_context_aware_candidates_from_legacy( regex = self._get_regex(dict_word) script = self._get_word_script(dict_word) - variant_text = variants["normalized"] for match in regex.finditer(variant_text): start = match.start() end = match.end() @@ -972,8 +990,8 @@ def _collect_context_aware_candidates_from_legacy( variant_text, start, end, - dict_word, - False, + match.group(0) if is_original else dict_word, + is_original, ) self._record_context_aware_match( text, @@ -986,6 +1004,48 @@ def _collect_context_aware_candidates_from_legacy( seen, ) + def _collect_context_aware_candidates_from_legacy_with_fallback( + self, + text: str, + variants: dict[str, str], + profane_words: list[str], + severity_map: dict[str, SeverityLevel], + matches: list[Match], + seen: set[str], + ) -> None: + self._collect_context_aware_candidates_from_legacy( + text, variants, profane_words, severity_map, matches, seen + ) + if profane_words: + return + + if variants["original"] != variants["normalized"]: + self._collect_context_aware_candidates_from_legacy( + text, + variants, + profane_words, + severity_map, + matches, + seen, + variant_key="original", + ) + if profane_words: + return + + if ( + variants["aggressive"] != variants["normalized"] + and variants["aggressive"] != variants["original"] + ): + self._collect_context_aware_candidates_from_legacy( + text, + variants, + profane_words, + severity_map, + matches, + seen, + variant_key="aggressive", + ) + def _build_context_aware_result( self, text: str, @@ -1056,7 +1116,7 @@ def _check_profanity_with_context_aware(self, text: str) -> CheckProfanityResult text, variants, profane_words, severity_map, matches, seen ) else: - self._collect_context_aware_candidates_from_legacy( + self._collect_context_aware_candidates_from_legacy_with_fallback( text, variants, profane_words, severity_map, matches, seen ) diff --git a/packages/py/tests/test_profane_words_collection.py b/packages/py/tests/test_profane_words_collection.py index 7e02f31..d0b6b78 100644 --- a/packages/py/tests/test_profane_words_collection.py +++ b/packages/py/tests/test_profane_words_collection.py @@ -108,3 +108,35 @@ def test_context_aware_whitelist_reason(self) -> None: assert result["profane_words"] == [] assert result["reason"] == "No profanity detected" assert context_filter.is_profane("This movie is the bomb") is False + + +class TestLegacyPathProfaneWords: + def test_word_boundaries_disabled_populates_fuzzy_words(self) -> None: + legacy = Filter( + { + "languages": ["english"], + "word_boundaries": False, + "fuzzy_tolerance_level": 0.6, + "disable_aho_corasick": True, + } + ) + result = legacy.check_profanity("This movie is the bomb") + assert result["contains_profanity"] is True + assert len(result["profane_words"]) > 0 + assert legacy.is_profane("This movie is the bomb") == result["contains_profanity"] + + def test_context_legacy_tier_fallback(self) -> None: + context_legacy = Filter( + { + "languages": ["english"], + "detect_leetspeak": True, + "enable_context_aware": True, + "disable_aho_corasick": True, + } + ) + for text in ("fuuuuuck", "f@ck"): + assert context_legacy.is_profane(text) is True + result = context_legacy.check_profanity(text) + assert result["contains_profanity"] is True + assert len(result["profane_words"]) > 0 + assert result["profane_words"] == [text] From f817818aefb6788e1f519b672769b8ec9b910487 Mon Sep 17 00:00:00 2001 From: wlike Date: Sat, 27 Jun 2026 18:59:02 +0800 Subject: [PATCH 05/11] fix: harden cache keys, config export, and PY/JS leetspeak parity Use config-aware result cache keys so runtime ignore/replace changes cannot return stale hits. Export full filter settings from get_config/getConfig, align Python leetspeak tables with JavaScript, and return legacy matches on the JS path. Co-authored-by: Cursor --- packages/js/src/filters/Filter.ts | 84 +++++++++++++++++-- packages/py/glin_profanity/filters/filter.py | 62 +++++++++++--- packages/py/glin_profanity/utils/leetspeak.py | 26 ++++++ packages/py/tests/test_filter_maintenance.py | 62 ++++++++++++++ 4 files changed, 212 insertions(+), 22 deletions(-) create mode 100644 packages/py/tests/test_filter_maintenance.py diff --git a/packages/js/src/filters/Filter.ts b/packages/js/src/filters/Filter.ts index 7d10bbb..9de73ff 100644 --- a/packages/js/src/filters/Filter.ts +++ b/packages/js/src/filters/Filter.ts @@ -48,6 +48,11 @@ class Filter { private confidenceThreshold: number; private contextAnalyzer?: ContextAnalyzer; private primaryLanguage: Language; + private allLanguages: boolean; + private languages: Language[]; + private customWords: string[]; + private disableAhoCorasick: boolean; + private domainWhitelists?: FilterConfig['domainWhitelists']; // Leetspeak and Unicode detection private detectLeetspeak: boolean; private leetspeakLevel: LeetspeakLevel; @@ -95,6 +100,11 @@ class Filter { this.contextWindow = config?.contextWindow ?? 3; this.confidenceThreshold = config?.confidenceThreshold ?? 0.7; this.primaryLanguage = config?.languages?.[0] || defaultLanguage; + this.allLanguages = config?.allLanguages ?? false; + this.languages = config?.languages ?? [defaultLanguage]; + this.customWords = config?.customWords ? [...config.customWords] : []; + this.disableAhoCorasick = config?.disableAhoCorasick ?? false; + this.domainWhitelists = config?.domainWhitelists; if (this.enableContextAware) { this.contextAnalyzer = new ContextAnalyzer({ @@ -418,6 +428,7 @@ class Filter { profaneSpans: Array<[string, number, number]>, severityMap: Record, isOriginalVariant: boolean, + matches?: Match[], ): void { const regex = this.getRegex(dictWord); const script = this.getWordScript(dictWord); @@ -443,6 +454,11 @@ class Filter { if (severityMap[resolved.matchedWord] === undefined) { severityMap[resolved.matchedWord] = severity; } + matches?.push({ + word: resolved.matchedWord, + index: resolved.start, + severity, + }); } } @@ -504,7 +520,9 @@ class Filter { */ public getConfig(): FilterConfig { return { - languages: [this.primaryLanguage], + languages: [...this.languages], + allLanguages: this.allLanguages, + customWords: [...this.customWords], caseSensitive: this.caseSensitive, wordBoundaries: this.wordBoundaries, replaceWith: this.replaceWith, @@ -516,15 +534,51 @@ class Filter { enableContextAware: this.enableContextAware, contextWindow: this.contextWindow, confidenceThreshold: this.confidenceThreshold, + domainWhitelists: this.domainWhitelists, detectLeetspeak: this.detectLeetspeak, leetspeakLevel: this.leetspeakLevel, normalizeUnicode: this.normalizeUnicodeEnabled, enableEvasionNormalization: this.enableEvasionNormalization, cacheResults: this.cacheResults, maxCacheSize: this.maxCacheSize, + disableAhoCorasick: this.disableAhoCorasick, }; } + private resultCacheKey(text: string): string { + const fingerprint = JSON.stringify({ + ignoreWords: Array.from(this.ignoreWords).sort(), + replaceWith: this.replaceWith ?? null, + wordBoundaries: this.wordBoundaries, + caseSensitive: this.caseSensitive, + detectLeetspeak: this.detectLeetspeak, + leetspeakLevel: this.leetspeakLevel, + normalizeUnicode: this.normalizeUnicodeEnabled, + enableEvasionNormalization: this.enableEvasionNormalization, + enableContextAware: this.enableContextAware, + contextWindow: this.contextWindow, + confidenceThreshold: this.confidenceThreshold, + fuzzyToleranceLevel: this.fuzzyToleranceLevel, + allowObfuscatedMatch: this.allowObfuscatedMatch, + severityLevels: this.severityLevels, + wordCount: this.words.size, + }); + return `${fingerprint}\0${text}`; + } + + private static formatProfanityReason( + flagged: boolean, + profaneWordList: string[], + ): string { + if (!flagged) { + return 'No profanity detected'; + } + if (profaneWordList.length > 0) { + return `Found ${profaneWordList.length} potential profanity matches`; + } + return 'Profanity detected'; + } + /** * Returns the current word dictionary size. * Useful for monitoring and debugging. @@ -873,6 +927,7 @@ class Filter { isOriginalVariant: boolean, profaneSpans: Array<[string, number, number]>, severityMap: Record, + matches: Match[], ): void { const variantText = variants[variantKey]; for (const dictWord of this.words.keys()) { @@ -887,6 +942,11 @@ class Filter { if (severityMap[dictWord] === undefined) { profaneSpans.push([dictWord, 0, dictWord.length]); severityMap[dictWord] = severity; + matches.push({ + word: dictWord, + index: 0, + severity, + }); } continue; } @@ -899,6 +959,7 @@ class Filter { profaneSpans, severityMap, isOriginalVariant, + matches, ); } } @@ -907,6 +968,7 @@ class Filter { const variants = this.getTextVariants(text, true); const profaneSpans: Array<[string, number, number]> = []; const severityMap: Record = {}; + const matches: Match[] = []; let containsProfanity = false; for (const dictWord of this.words.keys()) { @@ -924,6 +986,7 @@ class Filter { false, profaneSpans, severityMap, + matches, ); if (profaneSpans.length === 0 && containsProfanity) { if (variants.original !== variants.normalized) { @@ -934,6 +997,7 @@ class Filter { true, profaneSpans, severityMap, + matches, ); } if ( @@ -948,17 +1012,23 @@ class Filter { false, profaneSpans, severityMap, + matches, ); } } const resolved = this.profaneWordsFromSpans(profaneSpans, severityMap); - return this.buildProfanityResult( + const result = this.buildProfanityResult( text, resolved.profaneWords, resolved.severityMap, containsProfanity, ); + if (matches.length > 0) { + const dedupedWords = new Set(result.profaneWords); + result.matches = matches.filter((match) => dedupedWords.has(match.word)); + } + return result; } private recordContextAwareMatch( @@ -1281,9 +1351,7 @@ class Filter { ? [...matches].sort((a, b) => a.index - b.index || a.word.localeCompare(b.word)) : undefined, contextScore, - reason: flagged - ? `Found ${profaneWordList.length} potential profanity matches` - : 'No profanity detected', + reason: Filter.formatProfanityReason(flagged, profaneWordList), }; } @@ -1392,9 +1460,7 @@ class Filter { this.severityLevels && Object.keys(severityMap).length > 0 ? severityMap : undefined, - reason: flagged - ? `Found ${profaneWordList.length} potential profanity matches` - : 'No profanity detected', + reason: Filter.formatProfanityReason(flagged, profaneWordList), }; } @@ -1424,7 +1490,7 @@ class Filter { */ checkProfanity(text: string): CheckProfanityResult { // Check cache first - const cacheKey = text; + const cacheKey = this.resultCacheKey(text); const cachedResult = this.getFromCache(cacheKey); if (cachedResult) { this.debugLog('Cache hit for:', text.substring(0, 50)); diff --git a/packages/py/glin_profanity/filters/filter.py b/packages/py/glin_profanity/filters/filter.py index 29e84a7..fbcb249 100644 --- a/packages/py/glin_profanity/filters/filter.py +++ b/packages/py/glin_profanity/filters/filter.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import re from typing import Callable, Literal @@ -82,6 +83,11 @@ def __init__(self, config: FilterConfig | None = None) -> None: self.confidence_threshold = config.get("confidence_threshold", 0.7) languages = config.get("languages", ["english"]) self.primary_language: Language = languages[0] if languages else "english" + self._all_languages = config.get("all_languages", False) + self._languages: list[Language] = list(languages or ["english"]) + self._custom_words: list[str] = list(config.get("custom_words") or []) + self._disable_aho_corasick = bool(config.get("disable_aho_corasick", False)) + self._domain_whitelists = config.get("domain_whitelists") self.context_analyzer: ContextAnalyzer | None = None if self.enable_context_aware: domain_whitelists = config.get("domain_whitelists") or {} @@ -449,6 +455,9 @@ def get_config(self) -> FilterConfig: >>> # Later, restore: new_filter = Filter(json.load(open('filter.config.json'))) """ return { + "languages": list(self._languages), + "all_languages": self._all_languages, + "custom_words": list(self._custom_words), "case_sensitive": self.case_sensitive, "word_boundaries": self.word_boundaries, "replace_with": self.replace_with, @@ -460,14 +469,48 @@ def get_config(self) -> FilterConfig: "enable_context_aware": self.enable_context_aware, "context_window": self.context_window, "confidence_threshold": self.confidence_threshold, + "domain_whitelists": self._domain_whitelists, "detect_leetspeak": self.detect_leetspeak, "leetspeak_level": self.leetspeak_level, "normalize_unicode": self.normalize_unicode_enabled, "enable_evasion_normalization": self.enable_evasion_normalization, "cache_results": self.cache_results, "max_cache_size": self.max_cache_size, + "disable_aho_corasick": self._disable_aho_corasick, } + def _result_cache_key(self, text: str) -> str: + fingerprint = json.dumps( + { + "ignore_words": sorted(self.ignore_words), + "replace_with": self.replace_with, + "word_boundaries": self.word_boundaries, + "case_sensitive": self.case_sensitive, + "detect_leetspeak": self.detect_leetspeak, + "leetspeak_level": self.leetspeak_level, + "normalize_unicode": self.normalize_unicode_enabled, + "enable_evasion_normalization": self.enable_evasion_normalization, + "enable_context_aware": self.enable_context_aware, + "context_window": self.context_window, + "confidence_threshold": self.confidence_threshold, + "fuzzy_tolerance_level": self.fuzzy_tolerance_level, + "allow_obfuscated_match": self.allow_obfuscated_match, + "severity_levels": self.severity_levels, + "word_count": len(self.words), + }, + sort_keys=True, + separators=(",", ":"), + ) + return f"{fingerprint}\0{text}" + + @staticmethod + def _format_profanity_reason(flagged: bool, profane_word_list: list[str]) -> str: + if not flagged: + return "No profanity detected" + if profane_word_list: + return f"Found {len(profane_word_list)} potential profanity matches" + return "Profanity detected" + def get_word_count(self) -> int: """ Return the current word dictionary size. @@ -773,11 +816,7 @@ def _build_profanity_result( result: CheckProfanityResult = { "contains_profanity": flagged, "profane_words": profane_word_list, - "reason": ( - f"Found {len(profane_word_list)} potential profanity matches" - if flagged - else "No profanity detected" - ), + "reason": self._format_profanity_reason(flagged, profane_word_list), } if self.replace_with: @@ -1079,11 +1118,7 @@ def _build_context_aware_result( result: CheckProfanityResult = { "contains_profanity": flagged, "profane_words": profane_word_list, - "reason": ( - f"Found {len(profane_word_list)} potential profanity matches" - if flagged - else "No profanity detected" - ), + "reason": self._format_profanity_reason(flagged, profane_word_list), } if self.replace_with and profane_word_list: @@ -1301,7 +1336,8 @@ def check_profanity(self, text: str) -> CheckProfanityResult: True """ # Check cache first - cached_result = self._get_from_cache(text) + cache_key = self._result_cache_key(text) + cached_result = self._get_from_cache(cache_key) if cached_result is not None: self._debug_log("Cache hit for:", text[:50]) return cached_result @@ -1314,11 +1350,11 @@ def check_profanity(self, text: str) -> CheckProfanityResult: ) if result["contains_profanity"]: self._debug_log("Detected:", result.get("profane_words", [])) - self._add_to_cache(text, result) + self._add_to_cache(cache_key, result) return result result = self._check_profanity_with_context_aware(text) - self._add_to_cache(text, result) + self._add_to_cache(cache_key, result) return result def check_profanity_with_min_severity( diff --git a/packages/py/glin_profanity/utils/leetspeak.py b/packages/py/glin_profanity/utils/leetspeak.py index d6ba5b1..6844bcd 100644 --- a/packages/py/glin_profanity/utils/leetspeak.py +++ b/packages/py/glin_profanity/utils/leetspeak.py @@ -32,7 +32,16 @@ "{": "c", "[": "c", "+": "t", + "€": "e", + "&": "e", "#": "h", + "¥": "y", + "§": "s", + "†": "t", + "®": "r", + "©": "c", + "²": "2", + "³": "3", } # Aggressive single-character substitutions @@ -51,18 +60,35 @@ (r"\^", "a"), (r"\|3", "b"), (r"13", "b"), + (r"ß", "b"), (r"\|\)", "d"), (r"\|>", "d"), + (r"\[\)", "d"), (r"\|=", "f"), (r"ph", "f"), (r"\|-\|", "h"), (r"\}\{", "h"), (r"\|<", "k"), + (r"\|\{", "k"), (r"\|_", "l"), + (r"/\\/\\/", "m"), + (r"\|V\|", "m"), + (r"\[V\]", "m"), + (r"/\\/", "n"), + (r"\|\\\|", "n"), + (r"\|\*", "p"), + (r"\|o", "p"), (r"\|2", "r"), + (r"\|\?", "r"), (r"\|_\|", "u"), + (r"\\_\\", "u"), + (r"/_/", "u"), (r"\\/", "v"), + (r"\\/\\/", "w"), (r"vv", "w"), + (r"><", "x"), + (r"'/", "y"), + (r"7_", "z"), ] diff --git a/packages/py/tests/test_filter_maintenance.py b/packages/py/tests/test_filter_maintenance.py new file mode 100644 index 0000000..8c6238b --- /dev/null +++ b/packages/py/tests/test_filter_maintenance.py @@ -0,0 +1,62 @@ +"""Tests for filter cache, config export, and reason formatting.""" + +from glin_profanity import Filter + + +class TestResultCache: + def test_ignore_words_change_invalidates_cached_result(self) -> None: + filter_instance = Filter({"languages": ["english"], "cache_results": True}) + first = filter_instance.check_profanity("hello fuck world") + assert first["contains_profanity"] is True + + filter_instance.ignore_words.add("fuck") + second = filter_instance.check_profanity("hello fuck world") + assert second["contains_profanity"] is False + + def test_replace_with_change_invalidates_cached_processed_text(self) -> None: + filter_instance = Filter( + { + "languages": ["english"], + "cache_results": True, + "replace_with": "***", + } + ) + first = filter_instance.check_profanity("hello fuck world") + assert first["processed_text"] == "hello *** world" + + filter_instance.replace_with = "XXX" + second = filter_instance.check_profanity("hello fuck world") + assert second["processed_text"] == "hello XXX world" + + +class TestGetConfig: + def test_exports_round_trip_fields(self) -> None: + config = { + "languages": ["english", "spanish"], + "all_languages": False, + "custom_words": ["badword"], + "detect_leetspeak": True, + "disable_aho_corasick": True, + "domain_whitelists": {"english": ["clinical"]}, + } + exported = Filter(config).get_config() + assert exported["languages"] == ["english", "spanish"] + assert exported["custom_words"] == ["badword"] + assert exported["disable_aho_corasick"] is True + assert exported["domain_whitelists"] == {"english": ["clinical"]} + + +class TestReasonFormatting: + def test_flagged_without_words_uses_generic_reason(self) -> None: + filter_instance = Filter( + { + "languages": ["english"], + "word_boundaries": False, + "fuzzy_tolerance_level": 0.6, + "disable_aho_corasick": True, + } + ) + result = filter_instance.check_profanity("This movie is the bomb") + assert result["contains_profanity"] is True + assert result["profane_words"] + assert "potential profanity matches" in result["reason"] From 99f3a438375ca525cb3fd3f67a9720aec0810186 Mon Sep 17 00:00:00 2001 From: wlike Date: Mon, 29 Jun 2026 21:34:26 +0800 Subject: [PATCH 06/11] fix: align PY/JS profanity detection across CJK, emoji spans, and scanner edges Match Python combining-class semantics in JS variant mapping so emoji variation selectors no longer bleed into profane spans, and tighten CJK boundaries, accent folding, filter pool caching, and secret-pattern safety for consistent cross-language results. Co-authored-by: Cursor --- benchmarks/compare_csv_packages.py | 171 ++++++++++++-- packages/js/src/core/filterPool.ts | 23 +- packages/js/src/core/index.ts | 10 +- packages/js/src/data/dictionary.ts | 2 +- packages/js/src/filters/Filter.ts | 29 +++ .../js/src/filters/dictionaryAhoCorasick.ts | 19 +- packages/js/src/hooks/useProfanityChecker.ts | 6 +- packages/js/src/nlp/contextAnalyzer.ts | 8 +- packages/js/src/scanners/base.ts | 12 + packages/js/src/scanners/composite.ts | 5 +- .../scanners/patterns/injection-patterns.ts | 8 +- .../src/scanners/patterns/secret-patterns.ts | 94 ++++---- packages/js/src/scanners/pii.ts | 3 +- packages/js/src/scanners/prompt-injection.ts | 8 +- packages/js/src/scanners/secrets.ts | 3 +- packages/js/src/utils/combiningClass.ts | 210 ++++++++++++++++++ packages/js/src/utils/evasion.ts | 4 +- packages/js/src/utils/unicode.ts | 49 +++- packages/js/src/utils/variantMapping.ts | 41 +++- packages/js/src/utils/wordScript.ts | 85 ++++++- packages/js/tests/cjk-matching.test.ts | 27 ++- packages/js/tests/filter-pool.test.ts | 41 +++- packages/js/tests/leetspeak-unicode.test.ts | 44 ++++ packages/js/tests/scanners/composite.test.ts | 18 ++ .../tests/scanners/prompt-injection.test.ts | 11 + packages/js/tests/variant-mapping.test.ts | 40 ++++ .../py/glin_profanity/core/filter_pool.py | 50 +++-- packages/py/glin_profanity/data/dictionary.py | 13 +- .../filters/dictionary_aho_corasick.py | 6 +- packages/py/glin_profanity/filters/filter.py | 25 +++ packages/py/glin_profanity/scanners/base.py | 10 + .../scanners/patterns/injection_patterns.py | 6 +- .../scanners/patterns/secret_patterns.py | 98 ++++---- packages/py/glin_profanity/scanners/pii.py | 9 +- .../scanners/prompt_injection.py | 10 +- .../py/glin_profanity/scanners/secrets.py | 9 +- packages/py/glin_profanity/utils/unicode.py | 131 +++++++++-- .../glin_profanity/utils/variant_mapping.py | 32 ++- .../py/glin_profanity/utils/word_script.py | 77 ++++++- packages/py/tests/scanners/test_composite.py | 21 ++ .../tests/scanners/test_prompt_injection.py | 12 + .../py/tests/test_boundary_regressions.py | 80 +++++++ packages/py/tests/test_cjk_matching.py | 57 ++++- packages/py/tests/test_dictionary_lazy.py | 14 ++ packages/py/tests/test_evasion.py | 22 ++ packages/py/tests/test_filter_pool.py | 12 + .../py/tests/test_profane_words_collection.py | 97 +++++++- packages/py/tests/test_variant_mapping.py | 19 ++ .../{Norwegian.json => norwegian.json} | 0 tests/cross_language_parity_test.py | 19 +- 50 files changed, 1550 insertions(+), 250 deletions(-) create mode 100644 packages/js/src/utils/combiningClass.ts create mode 100644 packages/py/tests/test_boundary_regressions.py rename shared/dictionaries/{Norwegian.json => norwegian.json} (100%) diff --git a/benchmarks/compare_csv_packages.py b/benchmarks/compare_csv_packages.py index bc673f7..cc21d06 100644 --- a/benchmarks/compare_csv_packages.py +++ b/benchmarks/compare_csv_packages.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import sys import time from pathlib import Path @@ -18,37 +19,136 @@ "/Users/wlike/Downloads/glin_profanity_comparison_2026-06-24.xlsx" ) -FILTER_CONFIG = { - "all_languages": True, +# Override every language dictionary with the saylo curated word lists. +SAYLO_DICT_DIR = Path( + "/Users/wlike/Documents/saylo/saylo_dialog_safety/config/dictionaries" +) + +# Sentinel language key meaning "scan against every saylo dictionary at once". +ALL_LANGUAGES_KEY = "all" + +FILTER_CONFIG_BASE = { "detect_leetspeak": True, "normalize_unicode": True, } +def load_saylo_dictionaries() -> dict[str, list[str]]: + """Load every ``.json`` (a plain string array) from saylo.""" + if not SAYLO_DICT_DIR.exists(): + raise SystemExit(f"saylo dictionary dir not found: {SAYLO_DICT_DIR}") + dicts: dict[str, list[str]] = {} + for path in sorted(SAYLO_DICT_DIR.glob("*.json")): + with path.open(encoding="utf-8") as handle: + data = json.load(handle) + if not isinstance(data, list): + raise SystemExit(f"Unexpected format (need array) in {path}") + dicts[path.stem] = [str(word) for word in data] + if not dicts: + raise SystemExit(f"No dictionaries loaded from {SAYLO_DICT_DIR}") + print( + "Loaded saylo dictionaries: " + + ", ".join(f"{lang}={len(words)}" for lang, words in dicts.items()) + ) + return dicts + + +def words_for_language_key( + saylo_dicts: dict[str, list[str]], language_key: str +) -> list[str]: + """Return the saylo word list for a language key (or all combined). + + English profanity is overlaid onto every single-language dictionary because + cross-language chat texts very frequently contain English slurs (e.g. a + German/Italian sentence with "fuck"/"cock"); without this overlay those hits + are silently missed. + """ + if language_key == ALL_LANGUAGES_KEY: + combined: list[str] = [] + for words in saylo_dicts.values(): + combined.extend(words) + return combined + if language_key not in saylo_dicts: + # Saylo has no dictionary for this language: fall back to all words so we + # never silently under-detect. + combined = [] + for words in saylo_dicts.values(): + combined.extend(words) + return combined + words = list(saylo_dicts[language_key]) + if language_key != "english": + words.extend(saylo_dicts.get("english", [])) + return words + +# CSV locale tags -> glin-profanity dictionary language keys +LOCALE_TO_LANGUAGE: dict[str, str] = { + "en-US": "english", + "es-ES": "spanish", + "pt-BR": "portuguese", + "ja-JP": "japanese", + "zh-Hant-TW": "chinese", + "zh-Hans-CN": "chinese", + "fr-FR": "french", + "de-DE": "german", + "it-IT": "italian", +} + +PREFIX_TO_LANGUAGE: dict[str, str] = { + "en": "english", + "es": "spanish", + "pt": "portuguese", + "ja": "japanese", + "zh": "chinese", + "fr": "french", + "de": "german", + "it": "italian", +} + + +def locale_to_language(locale: object) -> str: + """Map CSV ``language`` column value to a dictionary language key. + + A missing/``-`` locale means "language unknown" -> scan with every + dictionary (``ALL_LANGUAGES_KEY``). + """ + raw = str(locale or "").strip() + if not raw or raw == "-": + return ALL_LANGUAGES_KEY + if raw in LOCALE_TO_LANGUAGE: + return LOCALE_TO_LANGUAGE[raw] + prefix = raw.split("-", 1)[0].lower() + return PREFIX_TO_LANGUAGE.get(prefix, ALL_LANGUAGES_KEY) + + def unload_glin_profanity() -> None: for name in list(sys.modules): if name == "glin_profanity" or name.startswith("glin_profanity."): del sys.modules[name] -def load_filter(package_parent: Path): +def load_filter(package_parent: Path, language_key: str, custom_words: list[str]): unload_glin_profanity() parent = str(package_parent.resolve()) sys.path = [p for p in sys.path if Path(p).resolve() != package_parent.resolve()] sys.path.insert(0, parent) from glin_profanity.filters.filter import Filter - return Filter(FILTER_CONFIG) + # languages=[] disables the bundled dictionaries; the saylo words are + # injected via custom_words so both packages run on the same vocabulary. + config = {**FILTER_CONFIG_BASE, "languages": [], "custom_words": custom_words} + return Filter(config) def run_batch( - label: str, package_parent: Path, texts: list[str] -) -> tuple[list[bool], list[str], list[frozenset[str]], list[float]]: + label: str, + package_parent: Path, + texts: list[str], + languages: list[str], + saylo_dicts: dict[str, list[str]], +) -> tuple[list[bool], list[str], list[frozenset[str]], list[float], list[str]]: print(f"Loading Filter from {package_parent} ({label})...") - t0 = time.perf_counter() - filt = load_filter(package_parent) - init_ms = (time.perf_counter() - t0) * 1000 - print(f" init: {init_ms:.1f} ms, words: {filt.get_word_count()}") + filters: dict[str, object] = {} + mapped_languages: list[str] = [] contains_list: list[bool] = [] words_list: list[str] = [] @@ -56,9 +156,21 @@ def run_batch( time_list: list[float] = [] total = len(texts) - for i, text in enumerate(texts): + for i, (text, locale) in enumerate(zip(texts, languages, strict=True)): if i > 0 and i % 1000 == 0: print(f" [{label}] {i}/{total}...") + language = locale_to_language(locale) + mapped_languages.append(language) + if language not in filters: + custom_words = words_for_language_key(saylo_dicts, language) + filters[language] = load_filter(package_parent, language, custom_words) + filt = filters[language] + print( + f" [{label}] cached language={language!r}, " + f"words={filt.get_word_count()}" + ) + filt = filters[language] + start = time.perf_counter() result = filt.check_profanity(text if isinstance(text, str) else str(text)) elapsed_ms = (time.perf_counter() - start) * 1000 @@ -69,7 +181,7 @@ def run_batch( words_list.append("; ".join(words)) time_list.append(elapsed_ms) - return contains_list, words_list, words_sets, time_list + return contains_list, words_list, words_sets, time_list, mapped_languages def classify_results_match( @@ -85,25 +197,31 @@ def classify_results_match( return "half" -def main() -> None: - if not CSV_PATH.exists(): - raise SystemExit(f"CSV not found: {CSV_PATH}") +def main(csv_path: Path = CSV_PATH, output_xlsx: Path = OUTPUT_XLSX) -> None: + if not csv_path.exists(): + raise SystemExit(f"CSV not found: {csv_path}") - print(f"Reading {CSV_PATH}...") - df = pd.read_csv(CSV_PATH, encoding="utf-8") + print(f"Reading {csv_path}...") + df = pd.read_csv(csv_path, encoding="utf-8") if "text" not in df.columns: raise SystemExit(f"Missing 'text' column. Columns: {list(df.columns)}") + if "language" not in df.columns: + raise SystemExit(f"Missing 'language' column. Columns: {list(df.columns)}") texts = df["text"].fillna("").astype(str).tolist() + locales = df["language"].fillna("").tolist() print(f"Rows: {len(texts)}") - v340_contains, v340_words, v340_word_sets, v340_times = run_batch( - "3.4.0", PACKAGE_V340, texts + saylo_dicts = load_saylo_dictionaries() + + v340_contains, v340_words, v340_word_sets, v340_times, dict_langs = run_batch( + "3.4.0", PACKAGE_V340, texts, locales, saylo_dicts ) - opt_contains, opt_words, opt_word_sets, opt_times = run_batch( - "feat-opt", PACKAGE_OPT, texts + opt_contains, opt_words, opt_word_sets, opt_times, _ = run_batch( + "feat-opt", PACKAGE_OPT, texts, locales, saylo_dicts ) + df["dictionary_language"] = dict_langs df["v340_contains_profanity"] = v340_contains df["v340_profane_words"] = v340_words df["v340_time_ms"] = v340_times @@ -156,8 +274,8 @@ def main() -> None: ] ) - print(f"Writing {OUTPUT_XLSX}...") - with pd.ExcelWriter(OUTPUT_XLSX, engine="openpyxl") as writer: + print(f"Writing {output_xlsx}...") + with pd.ExcelWriter(output_xlsx, engine="openpyxl") as writer: df.to_excel(writer, sheet_name="明细", index=False) summary.to_excel(writer, sheet_name="汇总", index=False) @@ -170,4 +288,9 @@ def main() -> None: if __name__ == "__main__": - main() + if len(sys.argv) > 1: + csv_arg = Path(sys.argv[1]) + out_arg = Path(sys.argv[2]) if len(sys.argv) > 2 else OUTPUT_XLSX + main(csv_arg, out_arg) + else: + main() diff --git a/packages/js/src/core/filterPool.ts b/packages/js/src/core/filterPool.ts index 6effe51..2b3c30d 100644 --- a/packages/js/src/core/filterPool.ts +++ b/packages/js/src/core/filterPool.ts @@ -6,12 +6,17 @@ import globalWhitelistData from '@shared/dictionaries/globalWhitelist.json'; const FILTER_POOL_MAX = 32; const filterPool = new Map(); +const GLOBAL_WHITELIST = (globalWhitelistData as { whitelist: string[] }).whitelist; +const GLOBAL_WHITELIST_SET = new Set(GLOBAL_WHITELIST); + export function createFilterConfig(config?: ProfanityCheckerConfig): FilterConfig { + const userIgnore = config?.ignoreWords ?? []; const effective: FilterConfig = { ...(config ?? {}), + // Idempotent: pre-merged configs must not duplicate global whitelist entries. ignoreWords: [ - ...(globalWhitelistData as { whitelist: string[] }).whitelist, - ...(config?.ignoreWords ?? []), + ...GLOBAL_WHITELIST, + ...userIgnore.filter((word) => !GLOBAL_WHITELIST_SET.has(word)), ], fuzzyToleranceLevel: config?.fuzzyToleranceLevel ?? 0.8, }; @@ -54,15 +59,21 @@ function normalizeConfigForKey(config: FilterConfig): FilterConfig { } function configCacheKey(config: FilterConfig): string { - return JSON.stringify(normalizeConfigForKey(config)); + const normalized = normalizeConfigForKey(config); + // Stable key regardless of object insertion order (matches Python sort_keys=True). + const sortedEntries = Object.keys(normalized) + .sort() + .map((key) => [key, normalized[key as keyof FilterConfig]] as const); + return JSON.stringify(Object.fromEntries(sortedEntries)); } /** * Returns a shared Filter instance for the given configuration. * Instances are evicted in FIFO order when the pool exceeds FILTER_POOL_MAX. */ -export function getPooledFilter(config: FilterConfig): Filter { - const key = configCacheKey(config); +export function getPooledFilter(config?: ProfanityCheckerConfig): Filter { + const effective = createFilterConfig(config); + const key = configCacheKey(effective); const existing = filterPool.get(key); if (existing) { filterPool.delete(key); @@ -70,7 +81,7 @@ export function getPooledFilter(config: FilterConfig): Filter { return existing; } - const filter = new Filter(config); + const filter = new Filter(effective); if (filterPool.size >= FILTER_POOL_MAX) { const oldestKey = filterPool.keys().next().value; if (oldestKey) { diff --git a/packages/js/src/core/index.ts b/packages/js/src/core/index.ts index ca2932b..11f09ac 100644 --- a/packages/js/src/core/index.ts +++ b/packages/js/src/core/index.ts @@ -1,11 +1,10 @@ import { ProfanityCheckerConfig, ProfanityCheckResult } from './types'; -import { createFilterConfig, getPooledFilter } from './filterPool'; +import { getPooledFilter } from './filterPool'; -export { clearFilterPool } from './filterPool'; +export { clearFilterPool, createFilterConfig } from './filterPool'; export function checkProfanity(text: string, config?: ProfanityCheckerConfig): ProfanityCheckResult { - const filterConfig = createFilterConfig(config); - const filter = getPooledFilter(filterConfig); + const filter = getPooledFilter(config); const checkResult = filter.checkProfanity(text); // Filter based on minSeverity (if provided) @@ -39,6 +38,5 @@ export async function checkProfanityAsync(text: string, config?: ProfanityChecke } export function isWordProfane(word: string, config?: ProfanityCheckerConfig): boolean { - const filter = getPooledFilter(createFilterConfig(config)); - return filter.isProfane(word); + return getPooledFilter(config).isProfane(word); } \ No newline at end of file diff --git a/packages/js/src/data/dictionary.ts b/packages/js/src/data/dictionary.ts index 3c2c8ae..97166b0 100644 --- a/packages/js/src/data/dictionary.ts +++ b/packages/js/src/data/dictionary.ts @@ -10,7 +10,7 @@ import German from '@shared/dictionaries/german.json'; import Hindi from '@shared/dictionaries/hindi.json'; import Hungarian from '@shared/dictionaries/hungarian.json'; import Korean from '@shared/dictionaries/korean.json'; -import Norwegian from '@shared/dictionaries/Norwegian.json'; +import Norwegian from '@shared/dictionaries/norwegian.json'; import Persian from '@shared/dictionaries/persian.json'; import Polish from '@shared/dictionaries/polish.json'; import Portuguese from '@shared/dictionaries/portuguese.json'; diff --git a/packages/js/src/filters/Filter.ts b/packages/js/src/filters/Filter.ts index 9de73ff..7f3e448 100644 --- a/packages/js/src/filters/Filter.ts +++ b/packages/js/src/filters/Filter.ts @@ -159,6 +159,15 @@ class Filter { words = [...words, ...config.customWords]; } + // Accent-folded aliases: the normalized text variant strips diacritics + // (normalizeUnicode), so an accented entry like "erección" would never + // match a user who typed "ereccion". Register the diacritic-free form as + // an extra alias so both spellings are caught. A length floor avoids short + // ambiguous folds (e.g. año -> ano, which would over-flag). + if (this.normalizeUnicodeEnabled) { + words = this.withAccentFoldedAliases(words); + } + this.words = new Map(); this.wordScripts = new Map(); const acWords: string[] = []; @@ -177,6 +186,26 @@ class Filter { : null; } + private static readonly ACCENT_ALIAS_MIN_LENGTH = 4; + + private withAccentFoldedAliases(words: string[]): string[] { + const seen = new Set(words.map((word) => word.toLowerCase())); + const extra: string[] = []; + for (const word of words) { + const folded = normalizeUnicode(word); + const foldedKey = folded.toLowerCase(); + if (foldedKey === word.toLowerCase() || seen.has(foldedKey)) { + continue; + } + if (foldedKey.replace(/ /g, '').length < Filter.ACCENT_ALIAS_MIN_LENGTH) { + continue; + } + seen.add(foldedKey); + extra.push(folded); + } + return [...words, ...extra]; + } + private shouldUseAhoCorasick(config?: FilterConfig): boolean { if (config?.disableAhoCorasick) { return false; diff --git a/packages/js/src/filters/dictionaryAhoCorasick.ts b/packages/js/src/filters/dictionaryAhoCorasick.ts index d0b60c6..d399ea1 100644 --- a/packages/js/src/filters/dictionaryAhoCorasick.ts +++ b/packages/js/src/filters/dictionaryAhoCorasick.ts @@ -14,9 +14,16 @@ type GraphemeSegmenterConstructor = new ( ) => GraphemeSegmenterInstance; const GraphemeSegmenter = ( - Intl as unknown as { Segmenter: GraphemeSegmenterConstructor } + Intl as unknown as { Segmenter?: GraphemeSegmenterConstructor } ).Segmenter; +if (typeof GraphemeSegmenter !== 'function') { + throw new Error( + 'glin-profanity requires Intl.Segmenter (Node 18+ or a modern browser) for ' + + 'Aho-Corasick dictionary matching. Upgrade your runtime or polyfill Intl.Segmenter.', + ); +} + export interface DictionaryMatch { dictWord: string; start: number; @@ -75,9 +82,11 @@ export class DictionaryAhoCorasick { private readonly wordGraphemeLengths: Map; constructor(words: string[]) { - this.ac = new AhoCorasick(words); + // Drop empty entries: an empty keyword would "match" at every position. + const nonEmpty = words.filter((word) => word.length > 0); + this.ac = new AhoCorasick(nonEmpty); this.wordGraphemeLengths = new Map( - words.map((word) => [word, countGraphemes(word)]), + nonEmpty.map((word) => [word, countGraphemes(word)]), ); } @@ -124,7 +133,9 @@ export class DictionaryAhoCorasick { const startGrapheme = endGraphemeIdx - wordLen + 1; const start = graphemeStartToStringIndex(text, startGrapheme); const end = graphemeEndToExclusiveStringIndex(text, endGraphemeIdx); - const script = options.wordScripts.get(dictWord) ?? 'latin'; + const script = + options.wordScripts.get(dictWord.toLowerCase()) ?? + classifyWordScript(dictWord); if ( !matchHasWordBoundary(text, start, end, script, options.wordBoundaries) diff --git a/packages/js/src/hooks/useProfanityChecker.ts b/packages/js/src/hooks/useProfanityChecker.ts index 82aa844..3373ec4 100644 --- a/packages/js/src/hooks/useProfanityChecker.ts +++ b/packages/js/src/hooks/useProfanityChecker.ts @@ -1,6 +1,6 @@ import { useState, useCallback, useRef, useEffect } from 'react'; import { checkProfanity, checkProfanityAsync } from '../core'; -import { createFilterConfig, getPooledFilter } from '../core/filterPool'; +import { getPooledFilter } from '../core/filterPool'; import type { ProfanityCheckerConfig } from '../core/types'; import type { CheckProfanityResult } from '../types/types'; @@ -8,10 +8,10 @@ export type { ProfanityCheckerConfig }; export const useProfanityChecker = (config?: ProfanityCheckerConfig) => { const [result, setResult] = useState(null); - const filterRef = useRef(getPooledFilter(createFilterConfig(config))); + const filterRef = useRef(getPooledFilter(config)); useEffect(() => { - filterRef.current = getPooledFilter(createFilterConfig(config)); + filterRef.current = getPooledFilter(config); }, [config]); const checkText = useCallback((text: string) => { diff --git a/packages/js/src/nlp/contextAnalyzer.ts b/packages/js/src/nlp/contextAnalyzer.ts index 1c7d9fa..514fe75 100644 --- a/packages/js/src/nlp/contextAnalyzer.ts +++ b/packages/js/src/nlp/contextAnalyzer.ts @@ -219,6 +219,12 @@ export class ContextAnalyzer { tokens: Array<{ word: string; start: number; end: number }>, charIndex: number, ): number { + // No tokens (e.g. text has no word characters): signal "not found" with -1 + // so the caller's fallback path runs, matching Python's _find_word_index. + if (tokens.length === 0) { + return -1; + } + for (let i = 0; i < tokens.length; i++) { const token = tokens[i]!; if (charIndex >= token.start && charIndex < token.end) { @@ -228,7 +234,7 @@ export class ContextAnalyzer { return Math.max(0, i - 1); } } - return Math.max(0, tokens.length - 1); + return tokens.length - 1; } private calculateSentimentScore(contextWords: string[], matchPosition: number): number { diff --git a/packages/js/src/scanners/base.ts b/packages/js/src/scanners/base.ts index 3ddc590..c2e004a 100644 --- a/packages/js/src/scanners/base.ts +++ b/packages/js/src/scanners/base.ts @@ -54,6 +54,18 @@ export interface ScanContext { strictness?: 'lenient' | 'moderate' | 'strict'; } +/** + * Coerce scanner input to a string. + * + * Scanners are reached from untyped runtime call sites; a non-string input + * (`null`, `undefined`, numbers, objects) would otherwise throw inside + * `RegExp.exec`. Treat anything that is not a string as empty so scanners never + * raise on bad input. Mirrors Python's `coerce_scan_input`. + */ +export function coerceScanInput(value: unknown): string { + return typeof value === 'string' ? value : ''; +} + /** * Build an ALLOW result for a scanner that found nothing to flag. * diff --git a/packages/js/src/scanners/composite.ts b/packages/js/src/scanners/composite.ts index 92b863c..9e2f597 100644 --- a/packages/js/src/scanners/composite.ts +++ b/packages/js/src/scanners/composite.ts @@ -72,7 +72,10 @@ export function scanAll(text: string, options?: ScanAllOptions): ScanResult[] { const active = new Set(requested); const vault = options?.vault; - const redact = vault !== undefined; + // Only redact when a real vault is provided. Treat an explicit null like + // Python's `vault is not None` (no redaction) rather than redacting with a + // placeholder. + const redact = vault !== undefined && vault !== null; const results: ScanResult[] = []; for (const name of ALL_SCANNERS) { diff --git a/packages/js/src/scanners/patterns/injection-patterns.ts b/packages/js/src/scanners/patterns/injection-patterns.ts index e76ec7c..fec6cb4 100644 --- a/packages/js/src/scanners/patterns/injection-patterns.ts +++ b/packages/js/src/scanners/patterns/injection-patterns.ts @@ -291,11 +291,9 @@ export const INJECTION_PATTERNS: InjectionPattern[] = [ }, { id: 'PI-036', - // Base64-looking blobs ≥40 chars that are not embedded in URLs or file paths. - // Uses a non-capturing boundary group instead of lookbehind for engine - // compatibility. Matches a non-path boundary char (or start-of-string anchor - // handled by alternation) before and after the blob. - pattern: /(?:^|[^a-zA-Z0-9/._-])([A-Za-z0-9+/]{40,}={0,2})(?:[^a-zA-Z0-9/._-]|$)/, + // Base64-looking blobs ≥40 chars, bounded to avoid ReDoS on long inputs. + // Lookbehind/lookahead match span excludes boundary chars (aligned with Python). + pattern: /(?(); diff --git a/packages/js/src/scanners/secrets.ts b/packages/js/src/scanners/secrets.ts index 8c83a1b..56e39de 100644 --- a/packages/js/src/scanners/secrets.ts +++ b/packages/js/src/scanners/secrets.ts @@ -5,7 +5,7 @@ */ import type { Scanner, ScanResult, ScanContext, ScanMatch } from './base'; -import { allowResult, blockResult } from './base'; +import { allowResult, blockResult, coerceScanInput } from './base'; import { SECRET_PATTERNS, type SecretPattern } from './patterns/secret-patterns'; import type { Vault } from './vault'; @@ -116,6 +116,7 @@ export class SecretsScanner implements Scanner { /** @inheritdoc */ scan(input: string, _ctx?: ScanContext): ScanResult { + input = coerceScanInput(input); const matches: ScanMatch[] = []; const reasons: string[] = []; let sanitized = input; diff --git a/packages/js/src/utils/combiningClass.ts b/packages/js/src/utils/combiningClass.ts new file mode 100644 index 0000000..f80462e --- /dev/null +++ b/packages/js/src/utils/combiningClass.ts @@ -0,0 +1,210 @@ +/** Canonical combining class != 0 (matches Python unicodedata.combining). */ +const NON_ZERO_COMBINING_RANGES: ReadonlyArray = [ + [0x300, 0x34E], + [0x350, 0x36F], + [0x483, 0x487], + [0x591, 0x5BD], + [0x5BF, 0x5BF], + [0x5C1, 0x5C2], + [0x5C4, 0x5C5], + [0x5C7, 0x5C7], + [0x610, 0x61A], + [0x64B, 0x65F], + [0x670, 0x670], + [0x6D6, 0x6DC], + [0x6DF, 0x6E4], + [0x6E7, 0x6E8], + [0x6EA, 0x6ED], + [0x711, 0x711], + [0x730, 0x74A], + [0x7EB, 0x7F3], + [0x7FD, 0x7FD], + [0x816, 0x819], + [0x81B, 0x823], + [0x825, 0x827], + [0x829, 0x82D], + [0x859, 0x85B], + [0x898, 0x89F], + [0x8CA, 0x8E1], + [0x8E3, 0x8FF], + [0x93C, 0x93C], + [0x94D, 0x94D], + [0x951, 0x954], + [0x9BC, 0x9BC], + [0x9CD, 0x9CD], + [0x9FE, 0x9FE], + [0xA3C, 0xA3C], + [0xA4D, 0xA4D], + [0xABC, 0xABC], + [0xACD, 0xACD], + [0xB3C, 0xB3C], + [0xB4D, 0xB4D], + [0xBCD, 0xBCD], + [0xC3C, 0xC3C], + [0xC4D, 0xC4D], + [0xC55, 0xC56], + [0xCBC, 0xCBC], + [0xCCD, 0xCCD], + [0xD3B, 0xD3C], + [0xD4D, 0xD4D], + [0xDCA, 0xDCA], + [0xE38, 0xE3A], + [0xE48, 0xE4B], + [0xEB8, 0xEBA], + [0xEC8, 0xECB], + [0xF18, 0xF19], + [0xF35, 0xF35], + [0xF37, 0xF37], + [0xF39, 0xF39], + [0xF71, 0xF72], + [0xF74, 0xF74], + [0xF7A, 0xF7D], + [0xF80, 0xF80], + [0xF82, 0xF84], + [0xF86, 0xF87], + [0xFC6, 0xFC6], + [0x1037, 0x1037], + [0x1039, 0x103A], + [0x108D, 0x108D], + [0x135D, 0x135F], + [0x1714, 0x1715], + [0x1734, 0x1734], + [0x17D2, 0x17D2], + [0x17DD, 0x17DD], + [0x18A9, 0x18A9], + [0x1939, 0x193B], + [0x1A17, 0x1A18], + [0x1A60, 0x1A60], + [0x1A75, 0x1A7C], + [0x1A7F, 0x1A7F], + [0x1AB0, 0x1ABD], + [0x1ABF, 0x1ACE], + [0x1B34, 0x1B34], + [0x1B44, 0x1B44], + [0x1B6B, 0x1B73], + [0x1BAA, 0x1BAB], + [0x1BE6, 0x1BE6], + [0x1BF2, 0x1BF3], + [0x1C37, 0x1C37], + [0x1CD0, 0x1CD2], + [0x1CD4, 0x1CE0], + [0x1CE2, 0x1CE8], + [0x1CED, 0x1CED], + [0x1CF4, 0x1CF4], + [0x1CF8, 0x1CF9], + [0x1DC0, 0x1DFF], + [0x20D0, 0x20DC], + [0x20E1, 0x20E1], + [0x20E5, 0x20F0], + [0x2CEF, 0x2CF1], + [0x2D7F, 0x2D7F], + [0x2DE0, 0x2DFF], + [0x302A, 0x302F], + [0x3099, 0x309A], + [0xA66F, 0xA66F], + [0xA674, 0xA67D], + [0xA69E, 0xA69F], + [0xA6F0, 0xA6F1], + [0xA806, 0xA806], + [0xA82C, 0xA82C], + [0xA8C4, 0xA8C4], + [0xA8E0, 0xA8F1], + [0xA92B, 0xA92D], + [0xA953, 0xA953], + [0xA9B3, 0xA9B3], + [0xA9C0, 0xA9C0], + [0xAAB0, 0xAAB0], + [0xAAB2, 0xAAB4], + [0xAAB7, 0xAAB8], + [0xAABE, 0xAABF], + [0xAAC1, 0xAAC1], + [0xAAF6, 0xAAF6], + [0xABED, 0xABED], + [0xFB1E, 0xFB1E], + [0xFE20, 0xFE2F], + [0x101FD, 0x101FD], + [0x102E0, 0x102E0], + [0x10376, 0x1037A], + [0x10A0D, 0x10A0D], + [0x10A0F, 0x10A0F], + [0x10A38, 0x10A3A], + [0x10A3F, 0x10A3F], + [0x10AE5, 0x10AE6], + [0x10D24, 0x10D27], + [0x10EAB, 0x10EAC], + [0x10EFD, 0x10EFF], + [0x10F46, 0x10F50], + [0x10F82, 0x10F85], + [0x11046, 0x11046], + [0x11070, 0x11070], + [0x1107F, 0x1107F], + [0x110B9, 0x110BA], + [0x11100, 0x11102], + [0x11133, 0x11134], + [0x11173, 0x11173], + [0x111C0, 0x111C0], + [0x111CA, 0x111CA], + [0x11235, 0x11236], + [0x112E9, 0x112EA], + [0x1133B, 0x1133C], + [0x1134D, 0x1134D], + [0x11366, 0x1136C], + [0x11370, 0x11374], + [0x11442, 0x11442], + [0x11446, 0x11446], + [0x1145E, 0x1145E], + [0x114C2, 0x114C3], + [0x115BF, 0x115C0], + [0x1163F, 0x1163F], + [0x116B6, 0x116B7], + [0x1172B, 0x1172B], + [0x11839, 0x1183A], + [0x1193D, 0x1193E], + [0x11943, 0x11943], + [0x119E0, 0x119E0], + [0x11A34, 0x11A34], + [0x11A47, 0x11A47], + [0x11A99, 0x11A99], + [0x11C3F, 0x11C3F], + [0x11D42, 0x11D42], + [0x11D44, 0x11D45], + [0x11D97, 0x11D97], + [0x11F41, 0x11F42], + [0x16AF0, 0x16AF4], + [0x16B30, 0x16B36], + [0x16FF0, 0x16FF1], + [0x1BC9E, 0x1BC9E], + [0x1D165, 0x1D169], + [0x1D16D, 0x1D172], + [0x1D17B, 0x1D182], + [0x1D185, 0x1D18B], + [0x1D1AA, 0x1D1AD], + [0x1D242, 0x1D244], + [0x1E000, 0x1E006], + [0x1E008, 0x1E018], + [0x1E01B, 0x1E021], + [0x1E023, 0x1E024], + [0x1E026, 0x1E02A], + [0x1E08F, 0x1E08F], + [0x1E130, 0x1E136], + [0x1E2AE, 0x1E2AE], + [0x1E2EC, 0x1E2EF], + [0x1E4EC, 0x1E4EF], + [0x1E8D0, 0x1E8D6], + [0x1E944, 0x1E94A], +]; + +export function hasNonZeroCombiningClass(char: string): boolean { + if (!char) return false; + const cp = char.codePointAt(0)!; + let lo = 0; + let hi = NON_ZERO_COMBINING_RANGES.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const [start, end] = NON_ZERO_COMBINING_RANGES[mid]!; + if (cp < start) hi = mid - 1; + else if (cp > end) lo = mid + 1; + else return true; + } + return false; +} diff --git a/packages/js/src/utils/evasion.ts b/packages/js/src/utils/evasion.ts index b2f71e3..dcda9a4 100644 --- a/packages/js/src/utils/evasion.ts +++ b/packages/js/src/utils/evasion.ts @@ -12,12 +12,12 @@ export function stripHtmlAndDecodeEntities(text: string): string { result = result.replace(/&#(\d+);/g, (_, dec: string) => { const code = parseInt(dec, 10); - return Number.isFinite(code) ? String.fromCharCode(code) : _; + return Number.isFinite(code) ? String.fromCodePoint(code) : _; }); result = result.replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) => { const code = parseInt(hex, 16); - return Number.isFinite(code) ? String.fromCharCode(code) : _; + return Number.isFinite(code) ? String.fromCodePoint(code) : _; }); return result diff --git a/packages/js/src/utils/unicode.ts b/packages/js/src/utils/unicode.ts index 7b4c6fb..47c3f4a 100644 --- a/packages/js/src/utils/unicode.ts +++ b/packages/js/src/utils/unicode.ts @@ -183,6 +183,43 @@ const ZERO_WIDTH_CHARS = [ '\u3164', // Hangul filler ]; +// Combining Japanese (han)dakuten. These are nonspacing marks (\p{Mn}) but +// encode a phonemic kana distinction, so they must survive diacritic folding. +const KANA_VOICED_SOUND_MARKS = new Set(['\u3099', '\u309A']); + +/** + * Return true for Basic Latin and Latin Extended letters (not Cyrillic/CJK/etc.). + */ +export function isLatinScriptLetter(char: string): boolean { + const code = char.codePointAt(0); + if (code === undefined || !/\p{L}/u.test(char)) { + return false; + } + return (code >= 0x41 && code <= 0x24f) || (code >= 0x1e00 && code <= 0x1eff); +} + +/** + * Skip homoglyph and leetspeak normalization when text contains non-Latin letters. + */ +export function textShouldSkipLatinObfuscationNormalization(text: string): boolean { + let latinLetters = 0; + let nonLatinLetters = 0; + for (const char of text) { + if (!/\p{L}/u.test(char)) { + continue; + } + if (isLatinScriptLetter(char)) { + latinLetters += 1; + } else { + nonLatinLetters += 1; + } + } + if (nonLatinLetters === 0) { + return false; + } + return nonLatinLetters >= latinLetters; +} + /** * Normalizes Unicode text for consistent profanity detection. * Handles various Unicode tricks used to evade filters. @@ -311,8 +348,16 @@ export function normalizeNFKD( let normalized = text.normalize('NFKD'); if (removeDiacritics) { - // Remove combining diacritical marks (U+0300 to U+036F) - normalized = normalized.replace(/[\u0300-\u036f]/g, ''); + // Remove nonspacing combining marks, but keep Japanese voiced/semi-voiced + // sound marks: ソ vs ゾ (and ハ/バ/パ) is a phonemic kana distinction, not a + // foldable diacritic. Dropping it makes クソ match ゾクゾク (ゾ→ソ) and ビッチ + // collapse to ヒッチ. Recompose with NFC afterwards so the kept marks fold + // back into precomposed kana (ソ + ゙ -> ゾ). + normalized = normalized.replace( + /\p{Mn}/gu, + (mark) => (KANA_VOICED_SOUND_MARKS.has(mark) ? mark : '') + ); + normalized = normalized.normalize('NFC'); } return normalized; diff --git a/packages/js/src/utils/variantMapping.ts b/packages/js/src/utils/variantMapping.ts index d4e6892..04d40ac 100644 --- a/packages/js/src/utils/variantMapping.ts +++ b/packages/js/src/utils/variantMapping.ts @@ -3,8 +3,10 @@ * Assumes normalization is order-preserving (no reordering of characters). */ +import { hasNonZeroCombiningClass } from './combiningClass'; import { AGGRESSIVE_SUBSTITUTIONS, MODERATE_SUBSTITUTIONS } from './leetspeak'; import { homoglyphToAscii } from './unicode'; +import { isUnicodeWordChar } from './wordScript'; export interface OriginalSpan { start: number; @@ -41,7 +43,7 @@ function normalizeCharForAlign(char: string): string { const decomposed = mapped.normalize('NFKD'); let base = ''; for (const codePoint of decomposed) { - if (!/\p{M}/u.test(codePoint)) { + if (!hasNonZeroCombiningClass(codePoint)) { base += codePoint; } } @@ -87,7 +89,7 @@ function isSkippableOriginalChar(char: string): boolean { } function isCombiningMark(char: string): boolean { - return /\p{M}/u.test(char); + return hasNonZeroCombiningClass(char); } function shouldTrimEdgePunctuation(char: string): boolean { @@ -152,11 +154,10 @@ function fallbackSpan( return finalizeSpan(original, idx, idx + needle.length); } - return finalizeSpan( - original, - Math.min(variantStart, original.length), - Math.min(variantEnd, original.length), - ); + // The variant word is not literally present in the original (e.g. it was + // reconstructed from a fully masked token like "f******"). Returning a + // guessed coordinate slice here only produces garbage spans, so drop it. + return { start: 0, end: 0, matchedText: '' }; } /** @@ -181,10 +182,14 @@ export function mapVariantSpanToOriginal( let variantIndex = 0; let origStart = -1; let origEnd = -1; + let lastAlignedNorm = ''; + let skippedSeparator = false; while (variantIndex < variant.length && originalIndex <= original.length) { if (variantIndex === variantStart && origStart === -1) { origStart = originalIndex; + lastAlignedNorm = ''; + skippedSeparator = false; } if (variantIndex === variantEnd) { origEnd = originalIndex; @@ -205,17 +210,33 @@ export function mapVariantSpanToOriginal( } if (charsAlign(originalChar, variantChar)) { + lastAlignedNorm = normalizeCharForAlign(originalChar) || originalChar.toLowerCase(); + skippedSeparator = false; variantIndex++; originalIndex++; continue; } if (isSkippableOriginalChar(originalChar)) { + originalIndex++; + skippedSeparator = true; + continue; + } + + // An extra original word character does not match the variant char. + // Only tolerate it when it is a collapsed repeat ("fuuuck" -> "fuck") + // that directly follows the previously aligned char with no separator + // in between. Otherwise the variant char was removed/masked in the + // original ("f******" -> "fuck") or positions drifted (NFKD punctuation + // like "…" -> ".."): stop and let the caller fall back to a literal + // lookup instead of consuming unrelated words. + const originalNorm = normalizeCharForAlign(originalChar) || originalChar.toLowerCase(); + if (!skippedSeparator && lastAlignedNorm && originalNorm === lastAlignedNorm) { originalIndex++; continue; } - originalIndex++; + break; } if (origEnd === -1 && variantIndex >= variantEnd) { @@ -293,10 +314,10 @@ export function isNestedProfaneSpan(shorter: string, longer: string): boolean { return false; } - const beforeOk = idx === 0 || !/\w/.test(longer[idx - 1]!); + const beforeOk = idx === 0 || !isUnicodeWordChar(longer[idx - 1]!); const afterIdx = idx + shorter.length; const afterOk = - afterIdx === longer.length || !/\w/.test(longer[afterIdx]!); + afterIdx === longer.length || !isUnicodeWordChar(longer[afterIdx]!); if (beforeOk && afterOk) { return true; diff --git a/packages/js/src/utils/wordScript.ts b/packages/js/src/utils/wordScript.ts index 2ea9458..d456f07 100644 --- a/packages/js/src/utils/wordScript.ts +++ b/packages/js/src/utils/wordScript.ts @@ -1,14 +1,23 @@ /** * Word-script classification and boundary checks for Latin vs CJK dictionary entries. * - * Latin entries use JavaScript `\b` semantics (ASCII word chars only). - * CJK entries use substring matching when word boundaries are enabled (no `\b`). + * Latin entries use Unicode-aware `\w` semantics (Python `\w` / JS `\b` with `u`). + * CJK entries use substring matching with a minimum grapheme length guard. */ export type WordScript = 'latin' | 'cjk'; -/** Mirrors JavaScript `\b` word character class without the `u` flag. */ -const LATIN_WORD_CHAR = /[A-Za-z0-9_]/; +/** Single-grapheme CJK hits require non-CJK neighbors (avoids 性 in 性格). */ +export const CJK_MIN_STANDALONE_GRAPHEMES = 2; + +/** + * Single CJK characters that are unambiguously profane regardless of neighbors. + * Unlike 性/骚/逼/淫/奸/賤/妓/尻/糞/裸 (which have common benign uses such as + * 性格/骚扰/逼近/淫雨/奸细/贫贱/芸妓/お尻/粪便/裸露), these characters effectively + * only occur in profane contexts, so we keep flagging them even when surrounded + * by other CJK characters (e.g. 挨肏, 肏她, 肏屄). + */ +export const UNAMBIGUOUS_CJK_SINGLE_PROFANITY = new Set('肏屄屌膣姦姘'); /** * Returns true when the code point belongs to a CJK-related script block. @@ -46,10 +55,29 @@ export function classifyWordScript(word: string): WordScript { return 'latin'; } -/** JavaScript `\b` boundary at index (between word and non-word ASCII chars). */ +/** Unicode-aware word character (Python `\w` / JS `\b` with `u` flag). */ +export function isUnicodeWordChar(char: string): boolean { + if (!char) { + return false; + } + if (char === '_') { + return true; + } + return /[\p{L}\p{N}]/u.test(char); +} + +/** + * Word boundary at index using Unicode-aware `\w` semantics. + * + * CJK neighbors are treated as non-word for Latin tokens: CJK has no spaces, + * so a Latin/pinyin token embedded in CJK (e.g. `jb` in `我的jb大`) should still + * be recognized as a standalone token rather than a continuation. + */ export function isLatinWordBoundaryBefore(text: string, index: number): boolean { - const leftIsWord = index > 0 && LATIN_WORD_CHAR.test(text[index - 1]!); - const rightIsWord = index < text.length && LATIN_WORD_CHAR.test(text[index]!); + const leftIsWord = + index > 0 && isUnicodeWordChar(text[index - 1]!) && !isCjkCharacter(text[index - 1]!); + const rightIsWord = + index < text.length && isUnicodeWordChar(text[index]!) && !isCjkCharacter(text[index]!); return leftIsWord !== rightIsWord; } @@ -59,12 +87,47 @@ export function hasLatinWordBoundary(text: string, start: number, end: number): ); } +function graphemeCountInSpan(text: string, start: number, end: number): number { + let count = 0; + let index = 0; + + while (index < text.length) { + const segStart = index; + index += 1; + while (index < text.length && /\p{M}/u.test(text[index]!)) { + index += 1; + } + if (index > start && segStart < end) { + count += 1; + } + if (segStart >= end) { + break; + } + } + return count; +} + /** - * CJK boundary: substring match anywhere (including ASCII-adjacent / wrapped text). - * Latin `\b` does not apply to CJK scripts; adjacency checks would miss obfuscation - * such as `hello他妈的`, `x乳x`, or `123エッチ456`. + * CJK boundary: multi-grapheme substring matches anywhere; single-grapheme + * matches require non-CJK neighbors (e.g. x乳x, hello操world). */ -export function hasCjkWordBoundary(_text: string, _start: number, _end: number): boolean { +export function hasCjkWordBoundary(text: string, start: number, end: number): boolean { + if (graphemeCountInSpan(text, start, end) >= CJK_MIN_STANDALONE_GRAPHEMES) { + return true; + } + + if (UNAMBIGUOUS_CJK_SINGLE_PROFANITY.has(text.slice(start, end))) { + return true; + } + + const before = start > 0 ? text[start - 1]! : ''; + const after = end < text.length ? text[end]! : ''; + if (before && isCjkCharacter(before)) { + return false; + } + if (after && isCjkCharacter(after)) { + return false; + } return true; } diff --git a/packages/js/tests/cjk-matching.test.ts b/packages/js/tests/cjk-matching.test.ts index 769df24..97c43de 100644 --- a/packages/js/tests/cjk-matching.test.ts +++ b/packages/js/tests/cjk-matching.test.ts @@ -22,10 +22,18 @@ describe('wordScript utilities', () => { expect(classifyWordScript('fuck')).toBe('latin'); }); - test('hasLatinWordBoundary matches JS \\b semantics', () => { + test('hasLatinWordBoundary matches Unicode \\w semantics', () => { expect(hasLatinWordBoundary('hello fuck world', 6, 10)).toBe(true); expect(hasLatinWordBoundary('scunthorpe', 5, 9)).toBe(false); expect(hasLatinWordBoundary('classic', 2, 5)).toBe(false); + expect(hasLatinWordBoundary('Ok cuántos quieres tener', 3, 5)).toBe(false); + expect(hasLatinWordBoundary('por fin adiós', 8, 11)).toBe(false); + }); + + test('hasLatinWordBoundary treats CJK neighbors as boundary', () => { + expect(hasLatinWordBoundary('我的jb大', 2, 4)).toBe(true); + expect(hasLatinWordBoundary('SM部屋', 0, 2)).toBe(true); + expect(hasLatinWordBoundary('passion', 1, 4)).toBe(false); }); test('hasCjkWordBoundary allows substring matches including ASCII adjacency', () => { @@ -34,6 +42,23 @@ describe('wordScript utilities', () => { expect(hasCjkWordBoundary('hello他妈的', 5, 8)).toBe(true); expect(hasCjkWordBoundary('123エッチ456', 3, 6)).toBe(true); }); + + test('hasCjkWordBoundary rejects single char inside compound', () => { + expect(hasCjkWordBoundary('性格', 0, 1)).toBe(false); + expect(hasCjkWordBoundary('明るい性格', 4, 5)).toBe(false); + expect(hasCjkWordBoundary('性', 0, 1)).toBe(true); + }); + + test('hasCjkWordBoundary allows unambiguous single profanity between CJK', () => { + expect(hasCjkWordBoundary('挨肏', 1, 2)).toBe(true); + expect(hasCjkWordBoundary('肏她', 0, 1)).toBe(true); + expect(hasCjkWordBoundary('被我肏屄的', 2, 3)).toBe(true); // 肏 + expect(hasCjkWordBoundary('被我肏屄的', 3, 4)).toBe(true); // 屄 + // Ambiguous single chars keep the strict neighbor rule. + expect(hasCjkWordBoundary('骚扰', 0, 1)).toBe(false); + expect(hasCjkWordBoundary('逼近', 0, 1)).toBe(false); + expect(hasCjkWordBoundary('淫雨', 0, 1)).toBe(false); + }); }); describe('CJK automatic matching strategy', () => { diff --git a/packages/js/tests/filter-pool.test.ts b/packages/js/tests/filter-pool.test.ts index cd62bc6..723f52e 100644 --- a/packages/js/tests/filter-pool.test.ts +++ b/packages/js/tests/filter-pool.test.ts @@ -1,4 +1,4 @@ -import { clearFilterPool, getPooledFilter } from '../src/core/filterPool'; +import { clearFilterPool, createFilterConfig, getPooledFilter } from '../src/core/filterPool'; describe('filterPool', () => { afterEach(() => { @@ -17,4 +17,43 @@ describe('filterPool', () => { const spanish = getPooledFilter({ languages: ['spanish'] }); expect(spanish).not.toBe(english); }); + + test('object field order does not affect pool key', () => { + const first = getPooledFilter({ + languages: ['english'], + customWords: ['word1'], + }); + const second = getPooledFilter({ + customWords: ['word1'], + languages: ['english'], + }); + expect(second).toBe(first); + }); + + test('pre-merged config shares the same pool instance', () => { + const raw = getPooledFilter({ + languages: ['english'], + ignoreWords: ['customword'], + }); + const premerged = getPooledFilter( + createFilterConfig({ + languages: ['english'], + ignoreWords: ['customword'], + }), + ); + expect(premerged).toBe(raw); + }); + + test('createFilterConfig is idempotent', () => { + const once = createFilterConfig({ ignoreWords: ['customword'] }); + const twice = createFilterConfig(once); + expect(twice.ignoreWords).toEqual(once.ignoreWords); + }); + + test('getPooledFilter merges global whitelist on raw config', () => { + const filter = getPooledFilter({ languages: ['english'] }); + // "Class" is in globalWhitelist.json and should not flag alone as profane + // when passed through the merged config path. + expect(filter.isProfane('Class')).toBe(false); + }); }); diff --git a/packages/js/tests/leetspeak-unicode.test.ts b/packages/js/tests/leetspeak-unicode.test.ts index 0f4531f..6af939a 100644 --- a/packages/js/tests/leetspeak-unicode.test.ts +++ b/packages/js/tests/leetspeak-unicode.test.ts @@ -27,6 +27,8 @@ describe('Evasion Normalization', () => { expect(stripHtmlAndDecodeEntities('shit')).toBe('shit'); expect(stripHtmlAndDecodeEntities('fuck')).toBe('fuck'); expect(stripHtmlAndDecodeEntities('a
ss')).toBe('ass'); + expect(stripHtmlAndDecodeEntities('😀')).toBe('😀'); + expect(stripHtmlAndDecodeEntities('😀')).toBe('😀'); }); }); @@ -132,6 +134,25 @@ describe('Unicode Normalization', () => { expect(normalizeUnicode('fосk')).toBe('fock'); // Cyrillic о expect(normalizeUnicode('fսck')).toBe('fuck'); // Armenian seh }); + + it('normalizes the extended homoglyph categories (kept in sync with Python)', () => { + expect(normalizeUnicode('у')).toBe('u'); // Cyrillic small u -> u (not y) + expect(normalizeUnicode('к')).toBe('k'); // Cyrillic small ka + expect(normalizeUnicode('ℝ')).toBe('R'); // double-struck capital R + expect(normalizeUnicode('ⓐ')).toBe('a'); // circled a + expect(normalizeUnicode('ʀ')).toBe('R'); // small-cap R + expect(normalizeUnicode('ɐ')).toBe('a'); // turned a + expect(normalizeUnicode('¢')).toBe('c'); // cent sign + }); + + it('keeps Japanese dakuten while folding Latin diacritics', () => { + // ゾ must not collapse to ソ (that made クソ match ゾクゾク); ビ stays ビ. + expect(normalizeUnicode('ゾクゾク')).toBe('ゾクゾク'); + expect(normalizeUnicode('ビッチ')).toBe('ビッチ'); + expect(normalizeUnicode('クソゲー')).toBe('クソゲー'); + expect(normalizeUnicode('café')).toBe('cafe'); + expect(normalizeUnicode('erección')).toBe('ereccion'); + }); }); describe('removeZeroWidthCharacters', () => { @@ -343,3 +364,26 @@ describe('checkProfanity with new options', () => { expect(result.profaneWords.length).toBeGreaterThan(0); }); }); + +describe('Accent-folded aliases', () => { + it('matches an accented entry even when the user drops diacritics', () => { + const filter = new Filter({ + languages: [], + customWords: ['erección'], + normalizeUnicode: true, + detectLeetspeak: true, + }); + expect(filter.checkProfanity('le muerdo la ereccion').containsProfanity).toBe(true); + expect(filter.checkProfanity('le muerdo la erección').containsProfanity).toBe(true); + }); + + it('does not alias short accented words (año -> ano)', () => { + const filter = new Filter({ + languages: [], + customWords: ['año'], + normalizeUnicode: true, + detectLeetspeak: true, + }); + expect(filter.checkProfanity('el ano').containsProfanity).toBe(false); + }); +}); diff --git a/packages/js/tests/scanners/composite.test.ts b/packages/js/tests/scanners/composite.test.ts index 6fbe328..2ce5cce 100644 --- a/packages/js/tests/scanners/composite.test.ts +++ b/packages/js/tests/scanners/composite.test.ts @@ -29,6 +29,24 @@ describe('scanAll — defaults', () => { } }); + test.each([null, undefined, 123, ['a'], { x: 1 }])( + 'non-string input %p does not crash', + (bad) => { + const results = scanAll(bad as unknown as string); + expect(results).toHaveLength(3); + for (const r of results) { + expect(r.decision).toBe('ALLOW'); + } + }, + ); + + test('vault null does not redact (matches Python is-not-None semantics)', () => { + const text = 'my email is user@example.com'; + const results = scanAll(text, { vault: null as never }); + const pii = results.find((r) => r.scanner === 'pii')!; + expect(pii.sanitized).toBe(text); + }); + test('PII text triggers pii scanner', () => { const results = scanAll('Contact me at user@example.com'); const piiResult = results.find((r) => r.scanner === 'pii')!; diff --git a/packages/js/tests/scanners/prompt-injection.test.ts b/packages/js/tests/scanners/prompt-injection.test.ts index a900bcc..256bd6a 100644 --- a/packages/js/tests/scanners/prompt-injection.test.ts +++ b/packages/js/tests/scanners/prompt-injection.test.ts @@ -107,6 +107,17 @@ describe('PromptInjectionScanner — HITL cases', () => { expect(result.score).toBeGreaterThan(0); expect(result.score).toBeLessThan(0.5); }); + + test('PI-036 match span excludes boundary delimiter chars', () => { + const blob = 'SGVsbG8gV29ybGQhIFRoaXMgaXMgYSB0ZXN0IG1lc3NhZ2UgZm9yIHNjYW5uaW5n'; + const text = `payload: ${blob}`; + const result = scanner.scan(text); + const pi036 = result.matches?.filter((m) => m.pattern === 'PI-036') ?? []; + expect(pi036).toHaveLength(1); + const matched = text.slice(pi036[0]!.startIndex, pi036[0]!.endIndex); + expect(matched).toBe(blob); + expect(matched.startsWith(':')).toBe(false); + }); }); // ─── Custom pattern injection ───────────────────────────────────────────────── diff --git a/packages/js/tests/variant-mapping.test.ts b/packages/js/tests/variant-mapping.test.ts index b713cb2..a2eddfa 100644 --- a/packages/js/tests/variant-mapping.test.ts +++ b/packages/js/tests/variant-mapping.test.ts @@ -60,10 +60,45 @@ describe('mapVariantSpanToOriginal', () => { expect(span.matchedText).toBe('camel toe'); }); + test('does not stretch a fully masked word across following words', () => { + // "f******" -> "fuck": the masked letters must not consume the rest of + // the sentence. The word is not literally recoverable, so the span drops. + const original = 'throat f****** myself while f****** your face'; + const variant = 'throat fuck myself while fuck your face'; + const span = mapVariantSpanToOriginal(original, variant, 7, 11); + expect(span.matchedText).toBe(''); + }); + + test('does not stretch span across unicode punctuation drift', () => { + // "…" -> ".." shifts indices; the mapped span for "cazzo" stays tight. + const original = 'amico…cazzo ci stavi'; + const variant = 'amico..cazzo ci stavi'; + const span = mapVariantSpanToOriginal(original, variant, 7, 12); + expect(span.matchedText).toBe('cazzo'); + }); + test('trims leading dots from gay span edges', () => { const span = trimProfaneSpanEdges('Sei un...gay?', 7, 12); expect(span.matchedText).toBe('gay'); }); + + test('maps profanity after emoji variation selector without FE0F in span', () => { + const original = '❤️fuck'; + const variant = '❤fuck'; + const span = mapVariantSpanToOriginal(original, variant, 1, 5); + expect(span.matchedText).toBe('fuck'); + expect(span.start).toBe(2); + expect(span.end).toBe(6); + }); + + test('maps profanity after keycap-style emoji without variation selector bleed', () => { + const original = '#️fuck'; + const variant = '#fuck'; + const span = mapVariantSpanToOriginal(original, variant, 1, 5); + expect(span.matchedText).toBe('fuck'); + expect(span.start).toBe(2); + expect(span.end).toBe(6); + }); }); describe('isNestedProfaneSpan', () => { @@ -74,4 +109,9 @@ describe('isNestedProfaneSpan', () => { test('does not treat ass in classic as nested', () => { expect(isNestedProfaneSpan('ass', 'classic')).toBe(false); }); + + test('uses Unicode word boundaries (matches Python \\w semantics)', () => { + // ß is a word character in Unicode \w; ass after ß is not word-bounded. + expect(isNestedProfaneSpan('ass', 'xßass')).toBe(false); + }); }); diff --git a/packages/py/glin_profanity/core/filter_pool.py b/packages/py/glin_profanity/core/filter_pool.py index 16bd9cd..386b3e0 100644 --- a/packages/py/glin_profanity/core/filter_pool.py +++ b/packages/py/glin_profanity/core/filter_pool.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import threading import warnings from collections import OrderedDict from pathlib import Path @@ -13,6 +14,7 @@ FILTER_POOL_MAX = 32 _filter_pool: OrderedDict[str, Filter] = OrderedDict() +_pool_lock = threading.Lock() _GLOBAL_WHITELIST_PATH = ( Path(__file__).resolve().parent.parent @@ -20,19 +22,31 @@ / "dictionaries" / "globalWhitelist.json" ) +_global_whitelist_cache: list[str] | None = None -def _load_global_whitelist() -> list[str]: - with _GLOBAL_WHITELIST_PATH.open(encoding="utf-8") as handle: - data = json.load(handle) - return list(data.get("whitelist", [])) +def _get_global_whitelist() -> list[str]: + """Return the shared global whitelist (loaded once per process).""" + global _global_whitelist_cache + if _global_whitelist_cache is None: + with _GLOBAL_WHITELIST_PATH.open(encoding="utf-8") as handle: + data = json.load(handle) + _global_whitelist_cache = list(data.get("whitelist", [])) + return _global_whitelist_cache def create_filter_config(config: FilterConfig | None = None) -> FilterConfig: """Build effective filter config, merging the shared global whitelist.""" effective: dict[str, Any] = dict(config or {}) - user_ignore = effective.get("ignore_words") or [] - effective["ignore_words"] = [*_load_global_whitelist(), *user_ignore] + user_ignore = list(effective.get("ignore_words") or []) + global_words = _get_global_whitelist() + global_set = set(global_words) + # Idempotent: calling create_filter_config on an already-merged config must + # not duplicate global whitelist entries (which would inflate the pool key). + effective["ignore_words"] = [ + *global_words, + *[word for word in user_ignore if word not in global_set], + ] effective.setdefault("fuzzy_tolerance_level", 0.8) if effective.get("allow_obfuscated_match") and effective.get("word_boundaries", True): @@ -78,20 +92,22 @@ def get_pooled_filter(config: FilterConfig | None = None) -> Filter: effective = create_filter_config(config) key = config_cache_key(effective) - existing = _filter_pool.get(key) - if existing is not None: - _filter_pool.move_to_end(key) - return existing + with _pool_lock: + existing = _filter_pool.get(key) + if existing is not None: + _filter_pool.move_to_end(key) + return existing - filter_instance = Filter(effective) - if len(_filter_pool) >= FILTER_POOL_MAX: - oldest_key = next(iter(_filter_pool)) - del _filter_pool[oldest_key] + filter_instance = Filter(effective) + if len(_filter_pool) >= FILTER_POOL_MAX: + oldest_key = next(iter(_filter_pool)) + del _filter_pool[oldest_key] - _filter_pool[key] = filter_instance - return filter_instance + _filter_pool[key] = filter_instance + return filter_instance def clear_filter_pool() -> None: """Clear all pooled Filter instances (intended for tests).""" - _filter_pool.clear() + with _pool_lock: + _filter_pool.clear() diff --git a/packages/py/glin_profanity/data/dictionary.py b/packages/py/glin_profanity/data/dictionary.py index 34ee73e..d1bbd56 100644 --- a/packages/py/glin_profanity/data/dictionary.py +++ b/packages/py/glin_profanity/data/dictionary.py @@ -70,14 +70,21 @@ def _load_dictionary(self, language: str) -> None: data = json.load(f) # Handle both {"words": [...]} and [...] formats if isinstance(data, dict) and "words" in data: - self._dictionaries[language] = data["words"] + raw_words = data["words"] elif isinstance(data, list): - self._dictionaries[language] = data + raw_words = data else: self._raise_format_error(filename) + return + # Guard against malformed entries (None/numbers/objects) so the + # AC build and matching never crash on bad dictionary data. + self._dictionaries[language] = [ + word for word in raw_words if isinstance(word, str) and word + ] except (FileNotFoundError, json.JSONDecodeError, ValueError) as e: print(f"Warning: Could not load {filename}: {e}") # noqa: T201 - self._dictionaries[language] = [] + # Do not cache the failure: a later call can retry after the file + # appears (e.g. misconfigured wheel path during startup). def get_words(self, language: Language) -> list[str]: """Get words for a specific language.""" diff --git a/packages/py/glin_profanity/filters/dictionary_aho_corasick.py b/packages/py/glin_profanity/filters/dictionary_aho_corasick.py index 5173dfc..464ec36 100644 --- a/packages/py/glin_profanity/filters/dictionary_aho_corasick.py +++ b/packages/py/glin_profanity/filters/dictionary_aho_corasick.py @@ -8,7 +8,11 @@ import ahocorasick -from glin_profanity.utils.word_script import WordScript, match_has_word_boundary +from glin_profanity.utils.word_script import ( + WordScript, + classify_word_script, + match_has_word_boundary, +) @dataclass(frozen=True) diff --git a/packages/py/glin_profanity/filters/filter.py b/packages/py/glin_profanity/filters/filter.py index fbcb249..eff90d2 100644 --- a/packages/py/glin_profanity/filters/filter.py +++ b/packages/py/glin_profanity/filters/filter.py @@ -136,6 +136,14 @@ def _load_words(self, config: FilterConfig) -> None: if custom_words: words.extend(custom_words) + # Accent-folded aliases: the normalized text variant strips diacritics + # (normalize_unicode), so an accented entry like "erección" would never + # match a user who typed "ereccion". Register the diacritic-free form as + # an extra alias so both spellings are caught. A length floor avoids + # short ambiguous folds (e.g. año -> ano, which would over-flag). + if self.normalize_unicode_enabled: + words = self._with_accent_folded_aliases(words) + # Store as set for faster lookup; track script per entry for boundary rules self.words: set[str] = {word.lower() for word in words} self.word_scripts: dict[str, WordScript] = {} @@ -151,6 +159,23 @@ def _load_words(self, config: FilterConfig) -> None: if self._should_use_aho_corasick(config): self.dictionary_matcher = DictionaryAhoCorasick(ac_words) + _ACCENT_ALIAS_MIN_LENGTH = 4 + + def _with_accent_folded_aliases(self, words: list[str]) -> list[str]: + """Append diacritic-stripped aliases for accented dictionary entries.""" + seen = {word.lower() for word in words} + extra: list[str] = [] + for word in words: + folded = normalize_unicode(word) + folded_key = folded.lower() + if folded_key == word.lower() or folded_key in seen: + continue + if len(folded_key.replace(" ", "")) < self._ACCENT_ALIAS_MIN_LENGTH: + continue + seen.add(folded_key) + extra.append(folded) + return words + extra + def _should_use_aho_corasick(self, config: FilterConfig) -> bool: if config.get("disable_aho_corasick"): return False diff --git a/packages/py/glin_profanity/scanners/base.py b/packages/py/glin_profanity/scanners/base.py index 5b4c42c..07f46a4 100644 --- a/packages/py/glin_profanity/scanners/base.py +++ b/packages/py/glin_profanity/scanners/base.py @@ -74,6 +74,16 @@ def scan(self, input: str, ctx: Optional[dict] = None) -> ScanResult: ... +def coerce_scan_input(value: object) -> str: + """Coerce scanner input to a string. + + Scanners receive arbitrary runtime values; a non-string input (``None``, + numbers, dicts) would otherwise blow up inside ``finditer``/``exec``. Treat + anything that is not a string as empty so scanners never raise on bad input. + """ + return value if isinstance(value, str) else "" + + def allow_result(scanner: str, input: str) -> ScanResult: """ Build an ALLOW result for a scanner that found nothing to flag. diff --git a/packages/py/glin_profanity/scanners/patterns/injection_patterns.py b/packages/py/glin_profanity/scanners/patterns/injection_patterns.py index 7d71d61..a3e2114 100644 --- a/packages/py/glin_profanity/scanners/patterns/injection_patterns.py +++ b/packages/py/glin_profanity/scanners/patterns/injection_patterns.py @@ -397,7 +397,7 @@ class InjectionPattern: ), InjectionPattern( id="PI-034", - pattern=re.compile(r"\bHUMAN\s*:\s*|ASSISTANT\s*:\s*"), + pattern=re.compile(r"\bHUMAN\s*:\s*|ASSISTANT\s*:\s*", _I), category="delimiter_injection", severity="medium", description="Anthropic/Claude-style conversation delimiter injection", @@ -416,9 +416,9 @@ class InjectionPattern: ), InjectionPattern( id="PI-036", - # Base64-looking blobs ≥40 chars that are not URLs or file paths + # Base64-looking blobs ≥40 chars, bounded to avoid ReDoS on long inputs. pattern=re.compile( - r"(? ScanResult: # noqa: A Returns: A ScanResult with decision, score, and match details. """ + input = coerce_scan_input(input) matches: list[ScanMatch] = [] reasons: list[str] = [] diff --git a/packages/py/glin_profanity/scanners/prompt_injection.py b/packages/py/glin_profanity/scanners/prompt_injection.py index cfdd37c..a891895 100644 --- a/packages/py/glin_profanity/scanners/prompt_injection.py +++ b/packages/py/glin_profanity/scanners/prompt_injection.py @@ -11,7 +11,14 @@ from typing import Optional -from .base import ScanDecision, ScanMatch, ScanResult, allow_result, block_result +from .base import ( + ScanDecision, + ScanMatch, + ScanResult, + allow_result, + block_result, + coerce_scan_input, +) from .patterns.injection_patterns import INJECTION_PATTERNS, InjectionPattern SEVERITY_WEIGHTS: dict[str, float] = { @@ -77,6 +84,7 @@ def scan(self, input: str, ctx: Optional[dict] = None) -> ScanResult: Returns: A :class:`~.base.ScanResult` with decision, score, and match details. """ + input = coerce_scan_input(input) effective_strictness = ( ctx.get("strictness", self._strictness) if ctx else self._strictness ) diff --git a/packages/py/glin_profanity/scanners/secrets.py b/packages/py/glin_profanity/scanners/secrets.py index 177afe3..247e403 100644 --- a/packages/py/glin_profanity/scanners/secrets.py +++ b/packages/py/glin_profanity/scanners/secrets.py @@ -9,7 +9,13 @@ import math from typing import Optional, TypedDict -from .base import ScanMatch, ScanResult, allow_result, block_result +from .base import ( + ScanMatch, + ScanResult, + allow_result, + block_result, + coerce_scan_input, +) from .patterns.secret_patterns import SECRET_PATTERNS, SecretPattern from .vault import Vault @@ -171,6 +177,7 @@ def scan(self, input: str, ctx: Optional[dict] = None) -> ScanResult: # noqa: A Returns: A ScanResult with decision, score, and match details. """ + input = coerce_scan_input(input) matches: list[ScanMatch] = [] reasons: list[str] = [] diff --git a/packages/py/glin_profanity/utils/unicode.py b/packages/py/glin_profanity/utils/unicode.py index 1d3385b..887bc0f 100644 --- a/packages/py/glin_profanity/utils/unicode.py +++ b/packages/py/glin_profanity/utils/unicode.py @@ -8,21 +8,25 @@ import unicodedata from typing import TypedDict -# Homoglyph mapping: visually similar Unicode characters to ASCII equivalents +# Homoglyph mapping: visually similar Unicode characters to ASCII equivalents. +# Kept in sync with packages/js/src/utils/unicode.ts (HOMOGLYPHS) so both +# implementations normalize identically. HOMOGLYPHS: dict[str, str] = { # Cyrillic homoglyphs (look like Latin) "а": "a", # Cyrillic small a "А": "A", # Cyrillic capital A "е": "e", # Cyrillic small e "Е": "E", # Cyrillic capital E + "к": "k", # Cyrillic small ka + "К": "K", # Cyrillic capital Ka "о": "o", # Cyrillic small o "О": "O", # Cyrillic capital O "р": "p", # Cyrillic small er "Р": "P", # Cyrillic capital Er "с": "c", # Cyrillic small es "С": "C", # Cyrillic capital Es - "у": "y", # Cyrillic small u - "У": "Y", # Cyrillic capital U + "у": "u", # Cyrillic small u (map to u, not y) + "У": "U", # Cyrillic capital U "х": "x", # Cyrillic small ha "Х": "X", # Cyrillic capital Ha "і": "i", # Cyrillic small i (Ukrainian) @@ -31,10 +35,13 @@ "Ј": "J", # Cyrillic capital Je "ѕ": "s", # Cyrillic small dze "Ѕ": "S", # Cyrillic capital Dze + # Currency and special symbols that look like letters + "¢": "c", # Cent sign + "ƒ": "f", # Latin small f with hook (florin) # Greek homoglyphs "α": "a", # Greek small alpha "Α": "A", # Greek capital Alpha - "β": "b", # Greek small beta + "β": "b", # Greek small beta (sort of) "Β": "B", # Greek capital Beta "ε": "e", # Greek small epsilon "Ε": "E", # Greek capital Epsilon @@ -53,29 +60,60 @@ "τ": "t", # Greek small tau "Τ": "T", # Greek capital Tau "υ": "u", # Greek small upsilon - "Ս": "U", # Armenian capital seh - "ս": "u", # Armenian small seh + "Ս": "U", # Armenian capital seh (looks like U) + "ս": "u", # Armenian small seh (looks like u) "Υ": "Y", # Greek capital Upsilon "χ": "x", # Greek small chi "Χ": "X", # Greek capital Chi + # Mathematical symbols + "ℂ": "C", # Double-struck capital C + "ℍ": "H", # Double-struck capital H + "ℕ": "N", # Double-struck capital N + "ℙ": "P", # Double-struck capital P + "ℚ": "Q", # Double-struck capital Q + "ℝ": "R", # Double-struck capital R + "ℤ": "Z", # Double-struck capital Z + # Subscript/superscript + "ᵃ": "a", "ᵇ": "b", "ᶜ": "c", "ᵈ": "d", "ᵉ": "e", + "ᶠ": "f", "ᵍ": "g", "ʰ": "h", "ⁱ": "i", "ʲ": "j", + "ᵏ": "k", "ˡ": "l", "ᵐ": "m", "ⁿ": "n", "ᵒ": "o", + "ᵖ": "p", "ʳ": "r", "ˢ": "s", "ᵗ": "t", "ᵘ": "u", + "ᵛ": "v", "ʷ": "w", "ˣ": "x", "ʸ": "y", "ᶻ": "z", + # Small caps + "ᴀ": "A", "ʙ": "B", "ᴄ": "C", "ᴅ": "D", "ᴇ": "E", + "ꜰ": "F", "ɢ": "G", "ʜ": "H", "ɪ": "I", "ᴊ": "J", + "ᴋ": "K", "ʟ": "L", "ᴍ": "M", "ɴ": "N", "ᴏ": "O", + "ᴘ": "P", "ǫ": "Q", "ʀ": "R", "ꜱ": "S", "ᴛ": "T", + "ᴜ": "U", "ᴠ": "V", "ᴡ": "W", "ʏ": "Y", "ᴢ": "Z", + # Circled letters + "ⓐ": "a", "ⓑ": "b", "ⓒ": "c", "ⓓ": "d", "ⓔ": "e", + "ⓕ": "f", "ⓖ": "g", "ⓗ": "h", "ⓘ": "i", "ⓙ": "j", + "ⓚ": "k", "ⓛ": "l", "ⓜ": "m", "ⓝ": "n", "ⓞ": "o", + "ⓟ": "p", "ⓠ": "q", "ⓡ": "r", "ⓢ": "s", "ⓣ": "t", + "ⓤ": "u", "ⓥ": "v", "ⓦ": "w", "ⓧ": "x", "ⓨ": "y", + "ⓩ": "z", + # Full-width letters + "ａ": "a", "ｂ": "b", "ｃ": "c", "ｄ": "d", "ｅ": "e", + "ｆ": "f", "ｇ": "g", "ｈ": "h", "ｉ": "i", "ｊ": "j", + "ｋ": "k", "ｌ": "l", "ｍ": "m", "ｎ": "n", "ｏ": "o", + "ｐ": "p", "ｑ": "q", "ｒ": "r", "ｓ": "s", "ｔ": "t", + "ｕ": "u", "ｖ": "v", "ｗ": "w", "ｘ": "x", "ｙ": "y", + "ｚ": "z", + # Mirrored/rotated + "ɐ": "a", "ɔ": "c", "ǝ": "e", "ɟ": "j", "ɥ": "h", + "ɯ": "m", "ɹ": "r", "ʇ": "t", "ʌ": "v", "ʍ": "w", # Common lookalikes - "ł": "l", - "Ł": "L", - "ø": "o", - "Ø": "O", - "đ": "d", - "Đ": "D", - "ħ": "h", - "Ħ": "H", - "ı": "i", - "İ": "I", - "ŋ": "n", - "Ŋ": "N", - "œ": "oe", - "Œ": "OE", + "ł": "l", "Ł": "L", + "ø": "o", "Ø": "O", + "đ": "d", "Đ": "D", + "ħ": "h", "Ħ": "H", + "ı": "i", "İ": "I", + "ĸ": "k", + "ŀ": "l", "Ŀ": "L", + "ŋ": "n", "Ŋ": "N", + "œ": "oe", "Œ": "OE", "ſ": "s", - "ŧ": "t", - "Ŧ": "T", + "ŧ": "t", "Ŧ": "T", } # Zero-width and invisible characters to remove @@ -97,6 +135,41 @@ "\u180E", # Mongolian vowel separator ] +# Combining Japanese (han)dakuten. These are nonspacing marks (category 'Mn') +# but encode a phonemic kana distinction, so they must survive diacritic folding. +_KANA_VOICED_SOUND_MARKS = frozenset(("\u3099", "\u309A")) + + +def is_latin_script_letter(char: str) -> bool: + """Return True for Basic Latin and Latin Extended letters (not Cyrillic/CJK/etc.).""" + if not char or not char.isalpha(): + return False + code = ord(char) + return code <= 0x024F or 0x1E00 <= code <= 0x1EFF + + +def text_should_skip_latin_obfuscation_normalization(text: str) -> bool: + """ + Skip homoglyph and leetspeak normalization when text contains non-Latin letters. + + Preserves Spanish/Portuguese accented Latin while avoiding Cyrillic→Latin + homoglyph false positives (e.g. поспал → pocpal matching ``oc``). + Mixed Latin obfuscation (e.g. ``fսck`` with one Armenian homoglyph) still + normalizes because Latin letters dominate. + """ + latin_letters = 0 + non_latin_letters = 0 + for char in text: + if not char.isalpha(): + continue + if is_latin_script_letter(char): + latin_letters += 1 + else: + non_latin_letters += 1 + if non_latin_letters == 0: + return False + return non_latin_letters >= latin_letters + def normalize_unicode( text: str, @@ -237,8 +310,18 @@ def normalize_nfkd(text: str, remove_diacritics: bool = True) -> str: normalized = unicodedata.normalize("NFKD", text) if remove_diacritics: - # Remove combining diacritical marks (category 'Mn' = Mark, Nonspacing) - normalized = "".join(c for c in normalized if unicodedata.category(c) != "Mn") + # Remove combining diacritical marks (category 'Mn' = Mark, Nonspacing), + # but keep Japanese voiced/semi-voiced sound marks: ソ vs ゾ (and ハ/バ/パ) + # is a phonemic distinction, not a foldable diacritic. Dropping it makes + # クソ match ゾクゾク (ゾ→ソ) and ビッチ collapse to ヒッチ. Recompose with + # NFC afterwards so the kept marks fold back into precomposed kana + # (ソ + ゙ -> ゾ). + normalized = "".join( + c + for c in normalized + if unicodedata.category(c) != "Mn" or c in _KANA_VOICED_SOUND_MARKS + ) + normalized = unicodedata.normalize("NFC", normalized) return normalized diff --git a/packages/py/glin_profanity/utils/variant_mapping.py b/packages/py/glin_profanity/utils/variant_mapping.py index a865a33..9157014 100644 --- a/packages/py/glin_profanity/utils/variant_mapping.py +++ b/packages/py/glin_profanity/utils/variant_mapping.py @@ -135,9 +135,10 @@ def _fallback_span( if idx >= 0: return _finalize_span(original, idx, idx + len(needle)) - start = min(variant_start, len(original)) - end = min(variant_end, len(original)) - return _finalize_span(original, start, end) + # The variant word is not literally present in the original (e.g. it was + # reconstructed from a fully masked token like "f******"). Returning a + # guessed coordinate slice here only produces garbage spans, so drop it. + return OriginalSpan(start=0, end=0, matched_text="") def map_variant_span_to_original( @@ -157,10 +158,14 @@ def map_variant_span_to_original( variant_index = 0 orig_start = -1 orig_end = -1 + last_aligned_norm = "" + skipped_separator = False while variant_index < len(variant) and original_index <= len(original): if variant_index == variant_start and orig_start == -1: orig_start = original_index + last_aligned_norm = "" + skipped_separator = False if variant_index == variant_end: orig_end = original_index break @@ -177,15 +182,34 @@ def map_variant_span_to_original( continue if _chars_align(original_char, variant_char): + last_aligned_norm = _normalize_char_for_align(original_char) or original_char.lower() + skipped_separator = False variant_index += 1 original_index += 1 continue if _is_skippable_original_char(original_char): original_index += 1 + skipped_separator = True continue - original_index += 1 + # An extra original word character does not match the variant char. + # Only tolerate it when it is a collapsed repeat ("fuuuck" -> "fuck") + # that directly follows the previously aligned char with no separator + # in between. Otherwise the variant char was removed/masked in the + # original ("f******" -> "fuck") or positions drifted (NFKD punctuation + # like "…" -> ".."): stop and let the caller fall back to a literal + # lookup instead of consuming unrelated words. + original_norm = _normalize_char_for_align(original_char) or original_char.lower() + if ( + not skipped_separator + and last_aligned_norm + and original_norm == last_aligned_norm + ): + original_index += 1 + continue + + break if orig_end == -1 and variant_index >= variant_end: orig_end = original_index diff --git a/packages/py/glin_profanity/utils/word_script.py b/packages/py/glin_profanity/utils/word_script.py index 3043f21..51b5c23 100644 --- a/packages/py/glin_profanity/utils/word_script.py +++ b/packages/py/glin_profanity/utils/word_script.py @@ -2,13 +2,20 @@ from __future__ import annotations -import re +import unicodedata from typing import Literal WordScript = Literal["latin", "cjk"] -# Mirrors JavaScript ``\\b`` word character class without the ``u`` flag. -_LATIN_WORD_CHAR = re.compile(r"[A-Za-z0-9_]") +# Single-grapheme CJK hits require non-CJK neighbors (avoids 性 in 性格). +CJK_MIN_STANDALONE_GRAPHEMES = 2 + +# Single CJK characters that are unambiguously profane regardless of neighbors. +# Unlike 性/骚/逼/淫/奸/賤/妓/尻/糞/裸 (which have common benign uses such as +# 性格/骚扰/逼近/淫雨/奸细/贫贱/芸妓/お尻/粪便/裸露), these characters effectively +# only occur in profane contexts, so we keep flagging them even when surrounded +# by other CJK characters (e.g. 挨肏, 肏她, 肏屄). +UNAMBIGUOUS_CJK_SINGLE_PROFANITY = frozenset("肏屄屌膣姦姘") def is_cjk_character(char: str) -> bool: @@ -43,10 +50,33 @@ def classify_word_script(word: str) -> WordScript: return "latin" +def is_unicode_word_char(char: str) -> bool: + """Unicode-aware word character (Python ``\\w`` / JS ``\\b`` with ``u`` flag).""" + if not char: + return False + if char == "_": + return True + category = unicodedata.category(char) + return category[0] in {"L", "N"} + + def is_latin_word_boundary_before(text: str, index: int) -> bool: - """JavaScript ``\\b`` boundary at index (between word and non-word ASCII chars).""" - left_is_word = index > 0 and bool(_LATIN_WORD_CHAR.match(text[index - 1])) - right_is_word = index < len(text) and bool(_LATIN_WORD_CHAR.match(text[index])) + """Word boundary at index using Unicode-aware ``\\w`` semantics. + + CJK neighbors are treated as non-word for Latin tokens: CJK has no spaces, + so a Latin/pinyin token embedded in CJK (e.g. ``jb`` in ``我的jb大``) should + still be recognized as a standalone token rather than a continuation. + """ + left_is_word = ( + index > 0 + and is_unicode_word_char(text[index - 1]) + and not is_cjk_character(text[index - 1]) + ) + right_is_word = ( + index < len(text) + and is_unicode_word_char(text[index]) + and not is_cjk_character(text[index]) + ) return left_is_word != right_is_word @@ -56,8 +86,39 @@ def has_latin_word_boundary(text: str, start: int, end: int) -> bool: ) -def has_cjk_word_boundary(_text: str, _start: int, _end: int) -> bool: - """Substring match anywhere, including ASCII-adjacent or digit-wrapped CJK.""" +def _grapheme_count_in_span(text: str, start: int, end: int) -> int: + count = 0 + index = 0 + length = len(text) + while index < length: + seg_start = index + index += 1 + while index < length and unicodedata.category(text[index]) in {"Mn", "Me", "Mc"}: + index += 1 + if index > start and seg_start < end: + count += 1 + if seg_start >= end: + break + return count + + +def has_cjk_word_boundary(text: str, start: int, end: int) -> bool: + """ + CJK boundary: multi-grapheme substring matches anywhere; single-grapheme + matches require non-CJK neighbors (e.g. x乳x, hello操world). + """ + if _grapheme_count_in_span(text, start, end) >= CJK_MIN_STANDALONE_GRAPHEMES: + return True + + if text[start:end] in UNAMBIGUOUS_CJK_SINGLE_PROFANITY: + return True + + before = text[start - 1] if start > 0 else "" + after = text[end] if end < len(text) else "" + if before and is_cjk_character(before): + return False + if after and is_cjk_character(after): + return False return True diff --git a/packages/py/tests/scanners/test_composite.py b/packages/py/tests/scanners/test_composite.py index 1a791ce..4de47bd 100644 --- a/packages/py/tests/scanners/test_composite.py +++ b/packages/py/tests/scanners/test_composite.py @@ -80,6 +80,27 @@ def test_unknown_scanner_raises(self) -> None: scan_all("Hello", scanners=["unknown_scanner"]) +# --------------------------------------------------------------------------- +# Robustness +# --------------------------------------------------------------------------- + + +class TestScanAllRobustness: + @pytest.mark.parametrize("bad", [None, 123, ["a"], {"x": 1}]) + def test_non_string_input_does_not_crash(self, bad: object) -> None: + results = scan_all(bad) # type: ignore[arg-type] + assert len(results) == 3 + for r in results: + assert r.decision == ScanDecision.ALLOW + + def test_vault_none_does_not_redact(self) -> None: + text = "my email is user@example.com" + results = scan_all(text, vault=None) + pii_result = next(r for r in results if r.scanner == "pii") + # No vault provided -> sanitized text must be left untouched. + assert pii_result.sanitized == text + + # --------------------------------------------------------------------------- # Vault sharing # --------------------------------------------------------------------------- diff --git a/packages/py/tests/scanners/test_prompt_injection.py b/packages/py/tests/scanners/test_prompt_injection.py index 77a20e0..78a13cc 100644 --- a/packages/py/tests/scanners/test_prompt_injection.py +++ b/packages/py/tests/scanners/test_prompt_injection.py @@ -126,6 +126,18 @@ def test_base64_blob_alone_is_low_severity(self) -> None: assert result.score > 0 assert result.score < 0.5 + def test_pi036_match_span_excludes_boundary_chars(self) -> None: + """PI-036 lookaround must not include leading/trailing delimiter chars.""" + blob = "SGVsbG8gV29ybGQhIFRoaXMgaXMgYSB0ZXN0IG1lc3NhZ2UgZm9yIHNjYW5uaW5n" + text = f"payload: {blob}" + scanner = PromptInjectionScanner(strictness="moderate") + result = scanner.scan(text) + pi036 = [m for m in result.matches if m.pattern == "PI-036"] + assert len(pi036) == 1 + matched = text[pi036[0].start_index : pi036[0].end_index] + assert matched == blob + assert not matched.startswith(":") and not matched.endswith(" ") + # ─── Custom pattern injection ───────────────────────────────────────────────── diff --git a/packages/py/tests/test_boundary_regressions.py b/packages/py/tests/test_boundary_regressions.py new file mode 100644 index 0000000..a1b9a18 --- /dev/null +++ b/packages/py/tests/test_boundary_regressions.py @@ -0,0 +1,80 @@ +"""Regression tests for boundary and normalization fixes (P0/P1/P2).""" + +from glin_profanity import Filter +from glin_profanity.utils.unicode import text_should_skip_latin_obfuscation_normalization + + +class TestLatinUnicodeBoundaries: + @classmethod + def setup_class(cls) -> None: + cls.filter = Filter( + { + "languages": ["spanish"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) + cls.portuguese_filter = Filter( + { + "languages": ["portuguese"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) + + def test_spanish_cu_ad_del_not_flagged(self) -> None: + cases = [ + "Ok cuántos quieres tener", + "No,porque ni siquiera te conozco no gracias*se acaba la cena*uy por fin adiós*me voy*", + "Aggg deberíamos de haber quedado de un modo en el que no viera tu estúpida cara", + "Te gusta? Cuánto? *Me acerco tomando sus manos*", + ] + for text in cases: + result = self.filter.check_profanity(text) + assert result["contains_profanity"] is False, text + + result = self.portuguese_filter.check_profanity( + "haaaa delícia disse gozando" + ) + assert result["contains_profanity"] is False + + +class TestNonLatinObfuscationSkip: + def test_russian_text_skips_homoglyph_leetspeak(self) -> None: + text = "Если ты это хочешь, то как бы и не против, но я все же поспал бы" + assert text_should_skip_latin_obfuscation_normalization(text) is True + result = Filter( + { + "languages": ["russian"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ).check_profanity(text) + assert result["contains_profanity"] is False + + def test_spanish_still_normalizes_latin_obfuscation(self) -> None: + text = "cuántos" + assert text_should_skip_latin_obfuscation_normalization(text) is False + + +class TestCjkMinLengthBoundary: + @classmethod + def setup_class(cls) -> None: + cls.filter = Filter( + { + "languages": ["japanese"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) + + def test_single_cjk_char_in_compound_not_flagged(self) -> None: + text = "はい、女として見てました。綺麗な顔立ちにそのスタイル、おまけに明るい性格で" + result = self.filter.check_profanity(text) + assert "性" not in result["profane_words"] + assert result["contains_profanity"] is False + + def test_standalone_cjk_profanity_still_detected(self) -> None: + chinese = Filter({"languages": ["chinese"]}) + assert chinese.is_profane("你他妈的") is True + assert chinese.is_profane("x乳x") is True diff --git a/packages/py/tests/test_cjk_matching.py b/packages/py/tests/test_cjk_matching.py index 14e6384..009bf6e 100644 --- a/packages/py/tests/test_cjk_matching.py +++ b/packages/py/tests/test_cjk_matching.py @@ -27,10 +27,19 @@ def test_classify_word_script_uses_characters_not_language(self) -> None: assert classify_word_script("sm") == "latin" assert classify_word_script("fuck") == "latin" - def test_has_latin_word_boundary_matches_js_b_semantics(self) -> None: + def test_has_latin_word_boundary_matches_unicode_w_semantics(self) -> None: assert has_latin_word_boundary("hello fuck world", 6, 10) is True assert has_latin_word_boundary("scunthorpe", 5, 9) is False assert has_latin_word_boundary("classic", 2, 5) is False + assert has_latin_word_boundary("Ok cuántos quieres tener", 3, 5) is False + assert has_latin_word_boundary("por fin adiós", 8, 11) is False + + def test_latin_boundary_treats_cjk_neighbors_as_boundary(self) -> None: + # Pinyin/abbrev profanity embedded in CJK is a standalone token. + assert has_latin_word_boundary("我的jb大", 2, 4) is True + assert has_latin_word_boundary("SM部屋", 0, 2) is True + # A Latin substring inside a Latin word is still not a boundary. + assert has_latin_word_boundary("passion", 1, 4) is False def test_has_cjk_word_boundary_allows_substring_including_ascii_adjacency( self, @@ -40,6 +49,23 @@ def test_has_cjk_word_boundary_allows_substring_including_ascii_adjacency( assert has_cjk_word_boundary("hello他妈的", 5, 8) is True assert has_cjk_word_boundary("123エッチ456", 3, 6) is True + def test_has_cjk_word_boundary_rejects_single_char_inside_compound(self) -> None: + assert has_cjk_word_boundary("性格", 0, 1) is False + assert has_cjk_word_boundary("明るい性格", 4, 5) is False + assert has_cjk_word_boundary("性", 0, 1) is True + + def test_has_cjk_word_boundary_allows_unambiguous_single_profanity(self) -> None: + # Unambiguous profane single chars match even between CJK neighbors. + assert has_cjk_word_boundary("挨肏", 1, 2) is True + assert has_cjk_word_boundary("肏她", 0, 1) is True + assert has_cjk_word_boundary("被我肏屄的", 2, 3) is True # 肏 + assert has_cjk_word_boundary("被我肏屄的", 3, 4) is True # 屄 + # Ambiguous single chars keep the strict neighbor rule (no false hits). + assert has_cjk_word_boundary("骚扰", 0, 1) is False + assert has_cjk_word_boundary("逼近", 0, 1) is False + assert has_cjk_word_boundary("淫雨", 0, 1) is False + assert has_cjk_word_boundary("性格", 0, 1) is False + class TestCjkAutomaticMatchingStrategy: @classmethod @@ -54,6 +80,14 @@ def test_chinese_detection_with_default_word_boundaries(self) -> None: assert self.chinese_filter.is_profane("你他妈的") is True assert len(self.chinese_filter.check_profanity("他妈的")["profane_words"]) > 0 + def test_unambiguous_single_char_detected_in_cjk_context(self) -> None: + assert self.chinese_filter.is_profane("挨肏") is True + assert self.chinese_filter.is_profane("肏她") is True + assert self.chinese_filter.check_profanity("肏她")["profane_words"] == ["肏"] + # Ambiguous single chars must not false-positive inside normal words. + assert self.chinese_filter.is_profane("受到骚扰") is False + assert self.chinese_filter.is_profane("性格很好") is False + def test_japanese_detection_with_default_word_boundaries(self) -> None: assert self.japanese_filter.is_profane("このエッチな話") is True assert self.japanese_filter.is_profane("エッチ") is True @@ -66,6 +100,27 @@ def test_japanese_ascii_entries_still_use_latin_boundaries(self) -> None: assert self.japanese_filter.is_profane("hello xx world") is True assert self.japanese_filter.is_profane("xxtra") is False + def test_latin_abbrev_in_cjk_context_detected(self) -> None: + f = Filter( + { + "languages": [], + "custom_words": ["jb"], + "normalize_unicode": True, + "detect_leetspeak": True, + } + ) + assert f.is_profane("我的jb大不大") is True + # Must not over-flag a Latin substring inside a Latin word. + ass = Filter( + { + "languages": [], + "custom_words": ["ass"], + "normalize_unicode": True, + "detect_leetspeak": True, + } + ) + assert ass.is_profane("passion fruit") is False + def test_detects_cjk_profanity_wrapped_in_or_adjacent_to_ascii(self) -> None: assert self.chinese_filter.is_profane("hello他妈的") is True assert self.chinese_filter.is_profane("x乳x") is True diff --git a/packages/py/tests/test_dictionary_lazy.py b/packages/py/tests/test_dictionary_lazy.py index b3d883e..222cc86 100644 --- a/packages/py/tests/test_dictionary_lazy.py +++ b/packages/py/tests/test_dictionary_lazy.py @@ -53,3 +53,17 @@ def test_global_instance_behavior(self) -> None: """ assert hasattr(dictionary, "LANGUAGE_FILES") assert len(dictionary.available_languages) == 24 + + def test_failed_load_is_not_cached(self, tmp_path) -> None: + """A missing dictionary file should not permanently cache an empty list.""" + loader = DictionaryLoader() + loader._dict_path = tmp_path + assert loader.get_words("english") == [] + assert "english" not in loader._dictionaries + + (tmp_path / "english.json").write_text( + '{"words": ["retryword"]}', encoding="utf-8" + ) + words = loader.get_words("english") + assert "retryword" in words + assert "english" in loader._dictionaries diff --git a/packages/py/tests/test_evasion.py b/packages/py/tests/test_evasion.py index a61c0a8..945e031 100644 --- a/packages/py/tests/test_evasion.py +++ b/packages/py/tests/test_evasion.py @@ -14,6 +14,9 @@ def test_strip_html_and_decode_entities(self) -> None: assert strip_html_and_decode_entities("shit") == "shit" assert strip_html_and_decode_entities("fuck") == "fuck" assert strip_html_and_decode_entities("a
ss") == "ass" + # Astral code points (Python chr handles these natively). + assert strip_html_and_decode_entities("😀") == "😀" + assert strip_html_and_decode_entities("😀") == "😀" def test_collapse_separated_characters(self) -> None: assert collapse_separated_characters("f.u.c.k") == "fuck" @@ -73,3 +76,22 @@ def test_does_not_flag_false_positive_traps(self) -> None: def test_armenian_homoglyph_normalizes_to_ascii(self) -> None: assert normalize_unicode("fսck") == "fuck" + + def test_extended_homoglyph_categories(self) -> None: + # Kept in sync with the JS HOMOGLYPHS table. + assert normalize_unicode("у") == "u" # Cyrillic small u -> u (not y) + assert normalize_unicode("к") == "k" # Cyrillic small ka + assert normalize_unicode("ℝ") == "R" # double-struck capital R + assert normalize_unicode("ⓐ") == "a" # circled a + assert normalize_unicode("ʀ") == "R" # small-cap R + assert normalize_unicode("ɐ") == "a" # turned a + assert normalize_unicode("¢") == "c" # cent sign + + def test_japanese_dakuten_survives_diacritic_folding(self) -> None: + # ゾ must not collapse to ソ (that made クソ match ゾクゾク); ビ stays ビ. + assert normalize_unicode("ゾクゾク") == "ゾクゾク" + assert normalize_unicode("ビッチ") == "ビッチ" + assert normalize_unicode("クソゲー") == "クソゲー" + # Latin diacritic folding must still work. + assert normalize_unicode("café") == "cafe" + assert normalize_unicode("erección") == "ereccion" diff --git a/packages/py/tests/test_filter_pool.py b/packages/py/tests/test_filter_pool.py index d5c1eef..047d2e6 100644 --- a/packages/py/tests/test_filter_pool.py +++ b/packages/py/tests/test_filter_pool.py @@ -62,3 +62,15 @@ def test_pooled_filter_behaves_like_direct_filter(self) -> None: assert pooled.check_profanity(text)["contains_profanity"] == direct.check_profanity( text )["contains_profanity"] + + def test_premerged_config_shares_same_pool_instance(self) -> None: + raw = get_pooled_filter({"languages": ["english"], "ignore_words": ["customword"]}) + premerged = get_pooled_filter( + create_filter_config({"languages": ["english"], "ignore_words": ["customword"]}) + ) + assert raw is premerged + + def test_create_filter_config_is_idempotent(self) -> None: + once = create_filter_config({"ignore_words": ["customword"]}) + twice = create_filter_config(once) + assert (once.get("ignore_words") or []) == (twice.get("ignore_words") or []) diff --git a/packages/py/tests/test_profane_words_collection.py b/packages/py/tests/test_profane_words_collection.py index d0b6b78..01b9ce8 100644 --- a/packages/py/tests/test_profane_words_collection.py +++ b/packages/py/tests/test_profane_words_collection.py @@ -15,7 +15,7 @@ class TestNormalizedProfaneWordsCollection: def setup_class(cls) -> None: cls.filter = Filter( { - "all_languages": True, + "languages": ["turkish"], "detect_leetspeak": True, "normalize_unicode": True, } @@ -27,6 +27,20 @@ def setup_class(cls) -> None: "normalize_unicode": True, } ) + cls.french_filter = Filter( + { + "languages": ["french"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) + cls.chinese_filter = Filter( + { + "languages": ["chinese"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) def test_amm_bno_q_haces(self) -> None: result = self.filter.check_profanity("Amm bno q haces") @@ -35,16 +49,16 @@ def test_amm_bno_q_haces(self) -> None: def test_cuéntame_only_reports_con(self) -> None: text = "Cuéntame algo *Digo con voz adormilada* Me gusta tu voz" - result = self.filter.check_profanity(text) + result = self.french_filter.check_profanity(text) assert "Cu" not in result["profane_words"] assert "con" in result["profane_words"] def test_f_dot_u_c_k_maps_to_original(self) -> None: - result = self.filter.check_profanity("say f.u.c.k off") + result = self.english_filter.check_profanity("say f.u.c.k off") assert result["profane_words"] == ["f.u.c.k"] def test_cjk_still_maps_to_original(self) -> None: - result = self.filter.check_profanity("123肏456") + result = self.chinese_filter.check_profanity("123肏456") assert result["contains_profanity"] is True assert result["profane_words"] == ["肏"] @@ -53,6 +67,31 @@ def test_f_at_ck_fallback_to_original(self) -> None: assert result["contains_profanity"] is True assert result["profane_words"] == ["f@ck"] + def test_masked_word_does_not_stretch_span(self) -> None: + # "f******" normalizes to "fuck"; the masked letters must not let the + # span run away across the rest of the sentence (mark-7 regression). + text = ( + "I shoved my ass back onto your fingers and then shove my cock " + "back down your throat f****** myself while f****** your face" + ) + result = self.english_filter.check_profanity(text) + assert result["contains_profanity"] is True + assert result["profane_words"] == ["cock", "ass"] + + def test_unicode_punctuation_shift_does_not_stretch_span(self) -> None: + # "…" -> ".." shifts positions; the span for "cazzo" must stay tight. + italian = Filter( + { + "languages": ["italian"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) + text = "ti giuro amico…cazzo ci stavi a fare tra un po mi tiravi un schiaffo" + result = italian.check_profanity(text) + assert result["contains_profanity"] is True + assert result["profane_words"] == ["cazzo"] + def test_repeated_chars_fallback_to_original(self) -> None: for text in ("fuuuuuck", "fffffffuck"): result = self.english_filter.check_profanity(text) @@ -60,8 +99,14 @@ def test_repeated_chars_fallback_to_original(self) -> None: assert result["profane_words"] == [text] def test_contains_implies_non_empty_profane_words(self) -> None: - for text in ("f@ck", "fuuuuuck", "say f.u.c.k off", "123肏456"): - result = self.english_filter.check_profanity(text) + cases = [ + (self.english_filter, "f@ck"), + (self.english_filter, "fuuuuuck"), + (self.english_filter, "say f.u.c.k off"), + (self.chinese_filter, "123肏456"), + ] + for filt, text in cases: + result = filt.check_profanity(text) if result["contains_profanity"]: assert len(result["profane_words"]) > 0 @@ -110,6 +155,33 @@ def test_context_aware_whitelist_reason(self) -> None: assert context_filter.is_profane("This movie is the bomb") is False +class TestAccentFoldedAliases: + def test_accented_entry_matches_diacritic_free_text(self) -> None: + f = Filter( + { + "languages": [], + "custom_words": ["erección"], + "normalize_unicode": True, + "detect_leetspeak": True, + } + ) + assert f.check_profanity("le muerdo la ereccion")["contains_profanity"] is True + assert f.check_profanity("le muerdo la erección")["contains_profanity"] is True + + def test_short_accented_fold_not_aliased(self) -> None: + # "año" folds to "ano" (length 3 < floor) so no alias is created, + # preventing año(year) from over-flagging as ano. + f = Filter( + { + "languages": [], + "custom_words": ["año"], + "normalize_unicode": True, + "detect_leetspeak": True, + } + ) + assert f.check_profanity("el ano")["contains_profanity"] is False + + class TestLegacyPathProfaneWords: def test_word_boundaries_disabled_populates_fuzzy_words(self) -> None: legacy = Filter( @@ -140,3 +212,16 @@ def test_context_legacy_tier_fallback(self) -> None: assert result["contains_profanity"] is True assert len(result["profane_words"]) > 0 assert result["profane_words"] == [text] + + def test_fuzzy_does_not_surface_unrelated_dict_substring(self) -> None: + fuzzy_filter = Filter( + { + "languages": ["english"], + "word_boundaries": False, + "fuzzy_tolerance_level": 0.6, + "disable_aho_corasick": True, + } + ) + text = "class bno session" + result = fuzzy_filter.check_profanity(text) + assert "bs" not in result["profane_words"] diff --git a/packages/py/tests/test_variant_mapping.py b/packages/py/tests/test_variant_mapping.py index c3376cb..f9f5c3b 100644 --- a/packages/py/tests/test_variant_mapping.py +++ b/packages/py/tests/test_variant_mapping.py @@ -62,6 +62,22 @@ def test_trims_leading_dots_from_gay(self) -> None: span = trim_profane_span_edges("Sei un...gay?", 7, 12) assert span.matched_text == "gay" + def test_maps_profane_word_after_emoji_variation_selector(self) -> None: + original = "❤️fuck" + variant = "❤fuck" + span = map_variant_span_to_original(original, variant, 1, 5) + assert span.matched_text == "fuck" + assert span.start == 2 + assert span.end == 6 + + def test_maps_profane_word_after_keycap_style_emoji(self) -> None: + original = "#️fuck" + variant = "#fuck" + span = map_variant_span_to_original(original, variant, 1, 5) + assert span.matched_text == "fuck" + assert span.start == 2 + assert span.end == 6 + class TestNestedProfaneSpan: def test_detects_nested_spans(self) -> None: @@ -69,3 +85,6 @@ def test_detects_nested_spans(self) -> None: def test_ignores_ass_inside_classic(self) -> None: assert not is_nested_profane_span("ass", "classic") + + def test_unicode_word_boundary_before_ass(self) -> None: + assert not is_nested_profane_span("ass", "xßass") diff --git a/shared/dictionaries/Norwegian.json b/shared/dictionaries/norwegian.json similarity index 100% rename from shared/dictionaries/Norwegian.json rename to shared/dictionaries/norwegian.json diff --git a/tests/cross_language_parity_test.py b/tests/cross_language_parity_test.py index a306c2c..831c1dc 100644 --- a/tests/cross_language_parity_test.py +++ b/tests/cross_language_parity_test.py @@ -15,15 +15,26 @@ REPO_ROOT = Path(__file__).resolve().parent.parent JS_ENTRY = REPO_ROOT / "packages" / "js" / "dist" / "index.cjs" +JS_PACKAGE = REPO_ROOT / "packages" / "js" sys.path.insert(0, str(REPO_ROOT / "packages" / "py")) from glin_profanity import Filter # noqa: E402 from glin_profanity.types.types import SeverityLevel # noqa: E402 -pytestmark = pytest.mark.skipif( - not JS_ENTRY.exists(), - reason="JavaScript dist not built; run `npm run build` in packages/js", -) + +@pytest.fixture(scope="session", autouse=True) +def ensure_js_dist_built() -> None: + if JS_ENTRY.exists(): + return + subprocess.run( + ["npm", "run", "build"], + cwd=JS_PACKAGE, + check=True, + capture_output=True, + text=True, + ) + if not JS_ENTRY.exists(): + pytest.fail("JavaScript dist build did not produce index.cjs") def python_config_to_js(config: dict[str, Any]) -> dict[str, Any]: From 45bf4a5bcaa2303738faa43d72476c1720efbcc1 Mon Sep 17 00:00:00 2001 From: wlike Date: Mon, 29 Jun 2026 22:02:23 +0800 Subject: [PATCH 07/11] fix: prevent 5m distance tokens from leetspeak sm false positives Skip digit substitutions in measurement-like tokens so 5m no longer normalizes to sm, and align contains_profanity with empty profane_words. Co-authored-by: Cursor --- packages/js/src/filters/Filter.ts | 10 +++++-- packages/js/src/utils/leetspeak.ts | 27 +++++++++++++++++-- packages/js/tests/leetspeak-unicode.test.ts | 21 +++++++++++++++ packages/py/glin_profanity/filters/filter.py | 4 +++ packages/py/glin_profanity/utils/leetspeak.py | 27 +++++++++++++++---- .../py/tests/test_boundary_regressions.py | 25 +++++++++++++++++ 6 files changed, 105 insertions(+), 9 deletions(-) diff --git a/packages/js/src/filters/Filter.ts b/packages/js/src/filters/Filter.ts index 7f3e448..e5d9b88 100644 --- a/packages/js/src/filters/Filter.ts +++ b/packages/js/src/filters/Filter.ts @@ -1362,10 +1362,13 @@ class Filter { contextScore = totalScore / matches.length; } - const flagged = + let flagged = containsProfanity !== undefined ? containsProfanity : profaneWordList.length > 0; + if (flagged && profaneWordList.length === 0) { + flagged = false; + } return { containsProfanity: flagged, @@ -1476,10 +1479,13 @@ class Filter { } } - const flagged = + let flagged = containsProfanity !== undefined ? containsProfanity : profaneWordList.length > 0; + if (flagged && profaneWordList.length === 0) { + flagged = false; + } return { containsProfanity: flagged, diff --git a/packages/js/src/utils/leetspeak.ts b/packages/js/src/utils/leetspeak.ts index 687049b..dcb7c59 100644 --- a/packages/js/src/utils/leetspeak.ts +++ b/packages/js/src/utils/leetspeak.ts @@ -178,14 +178,37 @@ const AGGRESSIVE_VOWEL_SUBSTITUTIONS: Record = { export { MODERATE_SUBSTITUTIONS, AGGRESSIVE_SUBSTITUTIONS }; +/** Digit + single Latin letter (e.g. 5m) — keep digits to avoid 5→s + m → "sm". */ +const MEASUREMENT_LIKE = /(? { + const skip = new Set(); + for (const match of text.matchAll(MEASUREMENT_LIKE)) { + const start = match.index ?? 0; + const end = start + match[0].length; + for (let index = start; index < end; index++) { + const char = text[index]; + if (char !== undefined && char >= '0' && char <= '9') { + skip.add(index); + } + } + } + return skip; +} + function applyCharSubstitutionsWithMap( text: string, substitutions: Record, ): string { + const skip = digitSubstitutionSkipIndices(text); const chars: string[] = []; for (let i = 0; i < text.length; i++) { - const char = text[i]; - chars.push(substitutions[char] ?? char); + const char = text[i]!; + if (substitutions[char] !== undefined && !skip.has(i)) { + chars.push(substitutions[char]!); + } else { + chars.push(char); + } } return chars.join(''); } diff --git a/packages/js/tests/leetspeak-unicode.test.ts b/packages/js/tests/leetspeak-unicode.test.ts index 6af939a..ec9fa60 100644 --- a/packages/js/tests/leetspeak-unicode.test.ts +++ b/packages/js/tests/leetspeak-unicode.test.ts @@ -69,6 +69,13 @@ describe('Leetspeak Detection', () => { expect(normalizeLeetspeak('f#ck', { level: 'moderate' })).toBe('fhck'); }); + it('should not treat measurement tokens like 5m as leetspeak', () => { + expect(normalizeLeetspeak('…5mまで', { level: 'moderate' })).toBe('…5mまで'); + expect(normalizeLeetspeak('stay 10m away', { level: 'moderate' })).toBe('stay 10m away'); + expect(normalizeLeetspeak('f4ck', { level: 'moderate' })).toBe('fack'); + expect(normalizeLeetspeak('5ex', { level: 'moderate' })).toBe('sex'); + }); + it('should handle aggressive substitutions', () => { expect(normalizeLeetspeak('ph4t', { level: 'aggressive' })).toBe('fat'); }); @@ -268,6 +275,20 @@ describe('Filter with Leetspeak and Unicode', () => { }); }); + describe('Measurement false positives', () => { + const filter = new Filter({ + languages: ['japanese'], + detectLeetspeak: true, + normalizeUnicode: true, + }); + + it('should not flag distance 5m as Japanese sm', () => { + const result = filter.checkProfanity('おい近づくな…5mまでだ'); + expect(result.containsProfanity).toBe(false); + expect(result.profaneWords).toEqual([]); + }); + }); + describe('Combined Leetspeak and Unicode', () => { const filter = new Filter({ languages: ['english'], diff --git a/packages/py/glin_profanity/filters/filter.py b/packages/py/glin_profanity/filters/filter.py index eff90d2..474e9e7 100644 --- a/packages/py/glin_profanity/filters/filter.py +++ b/packages/py/glin_profanity/filters/filter.py @@ -838,6 +838,8 @@ def _build_profanity_result( if contains_profanity is not None else len(profane_word_list) > 0 ) + if flagged and not profane_word_list: + flagged = False result: CheckProfanityResult = { "contains_profanity": flagged, "profane_words": profane_word_list, @@ -1140,6 +1142,8 @@ def _build_context_aware_result( if contains_profanity is not None else len(profane_word_list) > 0 ) + if flagged and not profane_word_list: + flagged = False result: CheckProfanityResult = { "contains_profanity": flagged, "profane_words": profane_word_list, diff --git a/packages/py/glin_profanity/utils/leetspeak.py b/packages/py/glin_profanity/utils/leetspeak.py index 6844bcd..ecf1a84 100644 --- a/packages/py/glin_profanity/utils/leetspeak.py +++ b/packages/py/glin_profanity/utils/leetspeak.py @@ -98,8 +98,28 @@ } +# Digit + single Latin letter (e.g. 5m, 10m) — keep digits to avoid 5→s + m → "sm". +_MEASUREMENT_LIKE = re.compile(r"(? frozenset[int]: + skip: set[int] = set() + for match in _MEASUREMENT_LIKE.finditer(text): + for index in range(match.start(), match.end()): + if text[index].isdigit(): + skip.add(index) + return frozenset(skip) + + def _apply_substitutions(text: str, substitutions: dict[str, str]) -> str: - return "".join(substitutions.get(char, char) for char in text) + skip = _digit_substitution_skip_indices(text) + result: list[str] = [] + for index, char in enumerate(text): + if char in substitutions and index not in skip: + result.append(substitutions[char]) + else: + result.append(char) + return "".join(result) def _apply_leetspeak_pre_collapse( @@ -190,10 +210,7 @@ def normalize_leetspeak( # Step 3: Apply single-character substitutions substitutions = _get_substitution_map(level) - result = [] - for char in normalized: - result.append(substitutions.get(char, char)) - normalized = "".join(result) + normalized = _apply_substitutions(normalized, substitutions) # Step 4: Collapse repeated characters (fuuuuck -> fuck) if collapse_repeated: diff --git a/packages/py/tests/test_boundary_regressions.py b/packages/py/tests/test_boundary_regressions.py index a1b9a18..fdce34c 100644 --- a/packages/py/tests/test_boundary_regressions.py +++ b/packages/py/tests/test_boundary_regressions.py @@ -78,3 +78,28 @@ def test_standalone_cjk_profanity_still_detected(self) -> None: chinese = Filter({"languages": ["chinese"]}) assert chinese.is_profane("你他妈的") is True assert chinese.is_profane("x乳x") is True + + +class TestMeasurementLeetspeakFalsePositive: + @classmethod + def setup_class(cls) -> None: + cls.filter = Filter( + { + "languages": ["japanese"], + "detect_leetspeak": True, + "normalize_unicode": True, + } + ) + + def test_distance_5m_not_flagged_as_sm(self) -> None: + text = "おい近づくな…5mまでだ" + result = self.filter.check_profanity(text) + assert result["contains_profanity"] is False + assert result["profane_words"] == [] + + def test_leetspeak_still_normalizes_embedded_digits(self) -> None: + from glin_profanity.utils.leetspeak import normalize_leetspeak + + assert normalize_leetspeak("f4ck") == "fack" + assert normalize_leetspeak("…5mまで") == "…5mまで" + assert normalize_leetspeak("5ex") == "sex" From f71542c7ecf2b90f81ebee85cc38094f1067aa85 Mon Sep 17 00:00:00 2001 From: wlike Date: Mon, 29 Jun 2026 22:15:53 +0800 Subject: [PATCH 08/11] chore: stop tracking local CSV comparison benchmark script Remove machine-specific compare_csv_packages.py from the repo and gitignore it so the script can stay in the local workspace only. Co-authored-by: Cursor --- .gitignore | 3 + benchmarks/compare_csv_packages.py | 296 ----------------------------- 2 files changed, 3 insertions(+), 296 deletions(-) delete mode 100644 benchmarks/compare_csv_packages.py diff --git a/.gitignore b/.gitignore index bc58bc9..d39bab5 100644 --- a/.gitignore +++ b/.gitignore @@ -241,3 +241,6 @@ CLAUDE.local.md # Roadmap kept local — not tracked upstream ROADMAP.md + +# Local CSV comparison benchmark (machine-specific paths) +benchmarks/compare_csv_packages.py diff --git a/benchmarks/compare_csv_packages.py b/benchmarks/compare_csv_packages.py deleted file mode 100644 index cc21d06..0000000 --- a/benchmarks/compare_csv_packages.py +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env python3 -"""Compare glin_profanity 3.4.0 vs feat-performance-opt on CSV text column.""" - -from __future__ import annotations - -import json -import sys -import time -from pathlib import Path - -import pandas as pd - -CSV_PATH = Path( - "/Users/wlike/Downloads/请求明细(safetyScore_ge_0.6)_2026-06-24_10_56_28.csv" -) -PACKAGE_V340 = Path("/Users/wlike/Downloads/glin_profanity-3.4.0") -PACKAGE_OPT = Path("/Users/wlike/Documents/learn/glin-profanity/packages/py") -OUTPUT_XLSX = Path( - "/Users/wlike/Downloads/glin_profanity_comparison_2026-06-24.xlsx" -) - -# Override every language dictionary with the saylo curated word lists. -SAYLO_DICT_DIR = Path( - "/Users/wlike/Documents/saylo/saylo_dialog_safety/config/dictionaries" -) - -# Sentinel language key meaning "scan against every saylo dictionary at once". -ALL_LANGUAGES_KEY = "all" - -FILTER_CONFIG_BASE = { - "detect_leetspeak": True, - "normalize_unicode": True, -} - - -def load_saylo_dictionaries() -> dict[str, list[str]]: - """Load every ``.json`` (a plain string array) from saylo.""" - if not SAYLO_DICT_DIR.exists(): - raise SystemExit(f"saylo dictionary dir not found: {SAYLO_DICT_DIR}") - dicts: dict[str, list[str]] = {} - for path in sorted(SAYLO_DICT_DIR.glob("*.json")): - with path.open(encoding="utf-8") as handle: - data = json.load(handle) - if not isinstance(data, list): - raise SystemExit(f"Unexpected format (need array) in {path}") - dicts[path.stem] = [str(word) for word in data] - if not dicts: - raise SystemExit(f"No dictionaries loaded from {SAYLO_DICT_DIR}") - print( - "Loaded saylo dictionaries: " - + ", ".join(f"{lang}={len(words)}" for lang, words in dicts.items()) - ) - return dicts - - -def words_for_language_key( - saylo_dicts: dict[str, list[str]], language_key: str -) -> list[str]: - """Return the saylo word list for a language key (or all combined). - - English profanity is overlaid onto every single-language dictionary because - cross-language chat texts very frequently contain English slurs (e.g. a - German/Italian sentence with "fuck"/"cock"); without this overlay those hits - are silently missed. - """ - if language_key == ALL_LANGUAGES_KEY: - combined: list[str] = [] - for words in saylo_dicts.values(): - combined.extend(words) - return combined - if language_key not in saylo_dicts: - # Saylo has no dictionary for this language: fall back to all words so we - # never silently under-detect. - combined = [] - for words in saylo_dicts.values(): - combined.extend(words) - return combined - words = list(saylo_dicts[language_key]) - if language_key != "english": - words.extend(saylo_dicts.get("english", [])) - return words - -# CSV locale tags -> glin-profanity dictionary language keys -LOCALE_TO_LANGUAGE: dict[str, str] = { - "en-US": "english", - "es-ES": "spanish", - "pt-BR": "portuguese", - "ja-JP": "japanese", - "zh-Hant-TW": "chinese", - "zh-Hans-CN": "chinese", - "fr-FR": "french", - "de-DE": "german", - "it-IT": "italian", -} - -PREFIX_TO_LANGUAGE: dict[str, str] = { - "en": "english", - "es": "spanish", - "pt": "portuguese", - "ja": "japanese", - "zh": "chinese", - "fr": "french", - "de": "german", - "it": "italian", -} - - -def locale_to_language(locale: object) -> str: - """Map CSV ``language`` column value to a dictionary language key. - - A missing/``-`` locale means "language unknown" -> scan with every - dictionary (``ALL_LANGUAGES_KEY``). - """ - raw = str(locale or "").strip() - if not raw or raw == "-": - return ALL_LANGUAGES_KEY - if raw in LOCALE_TO_LANGUAGE: - return LOCALE_TO_LANGUAGE[raw] - prefix = raw.split("-", 1)[0].lower() - return PREFIX_TO_LANGUAGE.get(prefix, ALL_LANGUAGES_KEY) - - -def unload_glin_profanity() -> None: - for name in list(sys.modules): - if name == "glin_profanity" or name.startswith("glin_profanity."): - del sys.modules[name] - - -def load_filter(package_parent: Path, language_key: str, custom_words: list[str]): - unload_glin_profanity() - parent = str(package_parent.resolve()) - sys.path = [p for p in sys.path if Path(p).resolve() != package_parent.resolve()] - sys.path.insert(0, parent) - from glin_profanity.filters.filter import Filter - - # languages=[] disables the bundled dictionaries; the saylo words are - # injected via custom_words so both packages run on the same vocabulary. - config = {**FILTER_CONFIG_BASE, "languages": [], "custom_words": custom_words} - return Filter(config) - - -def run_batch( - label: str, - package_parent: Path, - texts: list[str], - languages: list[str], - saylo_dicts: dict[str, list[str]], -) -> tuple[list[bool], list[str], list[frozenset[str]], list[float], list[str]]: - print(f"Loading Filter from {package_parent} ({label})...") - filters: dict[str, object] = {} - mapped_languages: list[str] = [] - - contains_list: list[bool] = [] - words_list: list[str] = [] - words_sets: list[frozenset[str]] = [] - time_list: list[float] = [] - - total = len(texts) - for i, (text, locale) in enumerate(zip(texts, languages, strict=True)): - if i > 0 and i % 1000 == 0: - print(f" [{label}] {i}/{total}...") - language = locale_to_language(locale) - mapped_languages.append(language) - if language not in filters: - custom_words = words_for_language_key(saylo_dicts, language) - filters[language] = load_filter(package_parent, language, custom_words) - filt = filters[language] - print( - f" [{label}] cached language={language!r}, " - f"words={filt.get_word_count()}" - ) - filt = filters[language] - - start = time.perf_counter() - result = filt.check_profanity(text if isinstance(text, str) else str(text)) - elapsed_ms = (time.perf_counter() - start) * 1000 - contains_list.append(bool(result.get("contains_profanity", False))) - words = result.get("profane_words") or [] - word_set = frozenset(words) - words_sets.append(word_set) - words_list.append("; ".join(words)) - time_list.append(elapsed_ms) - - return contains_list, words_list, words_sets, time_list, mapped_languages - - -def classify_results_match( - v340_contains: bool, - opt_contains: bool, - v340_words: frozenset[str], - opt_words: frozenset[str], -) -> str: - if v340_contains != opt_contains: - return "false" - if v340_words == opt_words: - return "true" - return "half" - - -def main(csv_path: Path = CSV_PATH, output_xlsx: Path = OUTPUT_XLSX) -> None: - if not csv_path.exists(): - raise SystemExit(f"CSV not found: {csv_path}") - - print(f"Reading {csv_path}...") - df = pd.read_csv(csv_path, encoding="utf-8") - if "text" not in df.columns: - raise SystemExit(f"Missing 'text' column. Columns: {list(df.columns)}") - if "language" not in df.columns: - raise SystemExit(f"Missing 'language' column. Columns: {list(df.columns)}") - - texts = df["text"].fillna("").astype(str).tolist() - locales = df["language"].fillna("").tolist() - print(f"Rows: {len(texts)}") - - saylo_dicts = load_saylo_dictionaries() - - v340_contains, v340_words, v340_word_sets, v340_times, dict_langs = run_batch( - "3.4.0", PACKAGE_V340, texts, locales, saylo_dicts - ) - opt_contains, opt_words, opt_word_sets, opt_times, _ = run_batch( - "feat-opt", PACKAGE_OPT, texts, locales, saylo_dicts - ) - - df["dictionary_language"] = dict_langs - df["v340_contains_profanity"] = v340_contains - df["v340_profane_words"] = v340_words - df["v340_time_ms"] = v340_times - df["opt_contains_profanity"] = opt_contains - df["opt_profane_words"] = opt_words - df["opt_time_ms"] = opt_times - df["results_match"] = [ - classify_results_match( - v340_contains[i], opt_contains[i], v340_word_sets[i], opt_word_sets[i] - ) - for i in range(len(texts)) - ] - - v340_avg = sum(v340_times) / len(v340_times) if v340_times else 0.0 - opt_avg = sum(opt_times) / len(opt_times) if opt_times else 0.0 - v340_total = sum(v340_times) - opt_total = sum(opt_times) - true_count = int((df["results_match"] == "true").sum()) - half_count = int((df["results_match"] == "half").sum()) - false_count = int((df["results_match"] == "false").sum()) - v340_flagged = sum(v340_contains) - opt_flagged = sum(opt_contains) - total_rows = len(df) - - summary = pd.DataFrame( - [ - {"metric": "total_rows", "value": total_rows}, - {"metric": "v340_avg_time_ms", "value": round(v340_avg, 4)}, - {"metric": "opt_avg_time_ms", "value": round(opt_avg, 4)}, - {"metric": "v340_total_time_ms", "value": round(v340_total, 2)}, - {"metric": "opt_total_time_ms", "value": round(opt_total, 2)}, - {"metric": "speedup_vs_v340", "value": round(v340_avg / opt_avg, 4) if opt_avg else None}, - {"metric": "v340_flagged_count", "value": v340_flagged}, - {"metric": "opt_flagged_count", "value": opt_flagged}, - {"metric": "results_match_true_count", "value": true_count}, - {"metric": "results_match_half_count", "value": half_count}, - {"metric": "results_match_false_count", "value": false_count}, - { - "metric": "results_match_true_rate_pct", - "value": round(100 * true_count / total_rows, 4) if total_rows else 0, - }, - { - "metric": "results_match_half_rate_pct", - "value": round(100 * half_count / total_rows, 4) if total_rows else 0, - }, - { - "metric": "results_match_false_rate_pct", - "value": round(100 * false_count / total_rows, 4) if total_rows else 0, - }, - ] - ) - - print(f"Writing {output_xlsx}...") - with pd.ExcelWriter(output_xlsx, engine="openpyxl") as writer: - df.to_excel(writer, sheet_name="明细", index=False) - summary.to_excel(writer, sheet_name="汇总", index=False) - - print("Done.") - print(f" v340 avg: {v340_avg:.4f} ms | opt avg: {opt_avg:.4f} ms") - print( - f" flagged: v340={v340_flagged} opt={opt_flagged} | " - f"match true={true_count} half={half_count} false={false_count}" - ) - - -if __name__ == "__main__": - if len(sys.argv) > 1: - csv_arg = Path(sys.argv[1]) - out_arg = Path(sys.argv[2]) if len(sys.argv) > 2 else OUTPUT_XLSX - main(csv_arg, out_arg) - else: - main() From f323888143b56cd893fff3a119503d4dc8171e98 Mon Sep 17 00:00:00 2001 From: wlike Date: Tue, 30 Jun 2026 11:43:27 +0800 Subject: [PATCH 09/11] chore: add release vs feat branch comparison benchmarks and report Automate lite shootout/test aggregation across release worktree and feat-performance-opt, and check in the comparison report with refreshed benchmark results. Co-authored-by: Cursor --- benchmarks/generate-comparison-report.mjs | 209 +++++++++--- benchmarks/optimization-comparison-lite.mjs | 5 +- benchmarks/optimization-comparison-lite.py | 6 +- benchmarks/optimization-comparison-report.md | 154 ++++++--- benchmarks/optimization-comparison.mjs | 5 +- benchmarks/optimization-comparison.py | 6 +- benchmarks/results/after-lite-js.json | 42 +-- benchmarks/results/after-lite-py.json | 42 +-- benchmarks/results/before-lite-js.json | 42 +-- benchmarks/results/before-lite-py.json | 42 +-- benchmarks/results/branch-comparison.json | 340 +++++++++++++++++++ benchmarks/run-branch-comparison.py | 274 +++++++++++++++ 12 files changed, 973 insertions(+), 194 deletions(-) create mode 100644 benchmarks/results/branch-comparison.json create mode 100755 benchmarks/run-branch-comparison.py diff --git a/benchmarks/generate-comparison-report.mjs b/benchmarks/generate-comparison-report.mjs index fe31807..d488d50 100644 --- a/benchmarks/generate-comparison-report.mjs +++ b/benchmarks/generate-comparison-report.mjs @@ -1,86 +1,187 @@ #!/usr/bin/env node -/** Generate markdown comparison from lite benchmark JSON files. */ +/** Generate markdown comparison report from branch-comparison.json + lite JSON. */ -import { readFileSync, writeFileSync } from 'node:fs'; +import { readFileSync, writeFileSync, existsSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); const dir = join(__dirname, 'results'); -function load(path) { - return JSON.parse(readFileSync(join(dir, path), 'utf8')); +function load(name) { + const path = join(dir, name); + if (!existsSync(path)) return null; + return JSON.parse(readFileSync(path, 'utf8')); } function pct(oldVal, newVal, invert = false) { - if (!oldVal || !newVal) return 'N/A'; + if (oldVal == null || newVal == null || oldVal === 0) return 'N/A'; let change = ((newVal - oldVal) / oldVal) * 100; if (invert) change = -change; const sign = change > 0 ? '+' : ''; return `${sign}${change.toFixed(1)}%`; } -function row(name, b, a) { - return { - name, - before_ops: b?.ops_per_sec, - after_ops: a?.ops_per_sec, - before_us: b?.avg_us, - after_us: a?.avg_us, - throughput: pct(b?.ops_per_sec, a?.ops_per_sec), - latency: pct(b?.avg_us, a?.avg_us, true), - }; -} - -function table(title, before, after, md) { +function benchTable(before, after) { const bMap = new Map(before.benchmarks.map((x) => [x.name, x])); const aMap = new Map(after.benchmarks.map((x) => [x.name, x])); const names = [...new Set([...bMap.keys(), ...aMap.keys()])]; - md += `## ${title}\n\n`; - md += `| 工作负载 | 优化前 ops/s | 优化后 ops/s | 吞吐变化 | 优化前 µs | 优化后 µs | 延迟变化 |\n`; - md += `|---------|-------------|-------------|---------|----------|----------|----------|\n`; + let md = `| 工作负载 | release ops/s | feat-opt ops/s | 吞吐变化 | release µs | feat-opt µs | 延迟变化 |\n`; + md += `|---------|--------------|---------------|---------|-----------|------------|----------|\n`; for (const name of names) { - const r = row(name, bMap.get(name), aMap.get(name)); - md += `| ${r.name} | ${r.before_ops?.toLocaleString() ?? '—'} | ${r.after_ops?.toLocaleString() ?? '—'} | ${r.throughput} | ${r.before_us ?? '—'} | ${r.after_us ?? '—'} | ${r.latency} |\n`; + const b = bMap.get(name); + const a = aMap.get(name); + md += `| ${name} | ${b?.ops_per_sec?.toLocaleString() ?? '—'} | ${a?.ops_per_sec?.toLocaleString() ?? '—'} | ${pct(b?.ops_per_sec, a?.ops_per_sec)} | ${b?.avg_us ?? '—'} | ${a?.avg_us ?? '—'} | ${pct(b?.avg_us, a?.avg_us, true)} |\n`; + } + return md + '\n'; +} + +function keyFullRows(before, after, keys) { + const bMap = new Map(before.benchmarks.map((x) => [x.name, x])); + const aMap = new Map(after.benchmarks.map((x) => [x.name, x])); + let md = `| Benchmark | release ops/s | feat-opt ops/s | 吞吐变化 | release µs | feat-opt µs |\n`; + md += `|-----------|--------------|---------------|---------|-----------|------------|\n`; + for (const name of keys) { + const b = bMap.get(name); + const a = aMap.get(name); + if (!b && !a) continue; + md += `| ${name} | ${b?.ops_per_sec?.toLocaleString() ?? '—'} | ${a?.ops_per_sec?.toLocaleString() ?? '—'} | ${pct(b?.ops_per_sec, a?.ops_per_sec)} | ${b?.avg_us ?? '—'} | ${a?.avg_us ?? '—'} |\n`; } - md += '\n'; - return md; + return md + '\n'; } +const meta = load('branch-comparison.json'); const beforeJs = load('before-lite-js.json'); const afterJs = load('after-lite-js.json'); const beforePy = load('before-lite-py.json'); const afterPy = load('after-lite-py.json'); +const fullBeforeJs = load('before-js.json'); +const fullAfterJs = load('after-js.json'); +const fullBeforePy = load('before-py.json'); +const fullAfterPy = load('after-py.json'); + +if (!beforeJs || !afterJs || !beforePy || !afterPy) { + console.error('Missing lite benchmark JSON in benchmarks/results/'); + process.exit(1); +} + +const releaseSha = meta?.release_sha ?? 'release'; +const featSha = meta?.feat_sha ?? 'feat-performance-opt'; +const commits = meta?.commits ?? []; + +let md = `# glin-profanity：release vs feat-performance-opt 完整对比报告\n\n`; +md += `_生成时间：${meta?.generated_at ?? new Date().toISOString()}_\n\n`; +md += `_基线分支：\`release\` @ \`${releaseSha}\`_\n\n`; +md += `_对比分支：\`feat-performance-opt\` @ \`${featSha}\`_\n\n`; +md += `_运行环境：Node ${beforeJs.node} / Python ${beforePy.python}_\n\n`; +md += `> **吞吐变化** 正数表示 feat-opt 更快；**延迟变化** 正数表示 feat-opt 延迟更低。\n\n`; + +md += `## 1. 分支差异概览\n\n`; +md += `| 指标 | 值 |\n|------|----|\n`; +md += `| release SHA | \`${releaseSha}\` |\n`; +md += `| feat-performance-opt SHA | \`${featSha}\` |\n`; +md += `| 新增 commit 数 | ${commits.length} |\n`; +md += `| 代码变更规模 | 91 files, +8795 / -827 lines（相对 release） |\n\n`; + +if (commits.length) { + md += `
feat-performance-opt 相对 release 的 commit 列表（${commits.length}）\n\n`; + for (const line of commits) md += `- ${line}\n`; + md += `\n
\n\n`; +} + +md += `## 2. 核心能力变更（feat-performance-opt）\n\n`; +md += `| 能力 | release | feat-performance-opt |\n|------|---------|---------------------|\n`; +md += `| Aho-Corasick 词典快路径 | 无 | 有（\`disable_aho_corasick\` 可回退 legacy） |\n`; +md += `| Evasion 归一化（HTML/掩码/分隔符） | 无 | 有 |\n`; +md += `| Context-aware 检测 | 基础 | 与 AC 路径对齐，parity 测试覆盖 |\n`; +md += `| CJK 边界 / 单字策略 | 较弱 | 方向 B 拉丁边界 + 无歧义单字白名单 |\n`; +md += `| Unicode homoglyph 表 | 较小 | Py/JS 189 条完全同步 |\n`; +md += `| Variant span 映射 | 基础 | emoji FE0F / combining class 对齐 |\n`; +md += `| Filter 实例池 | 无 | Py/JS \`get_pooled_filter\` / \`getPooledFilter\` |\n`; +md += `| 跨语言 parity 测试 | 无 | \`tests/cross_language_parity_test.py\` |\n\n`; + +if (meta?.tests) { + md += `## 3. 测试套件\n\n`; + md += `| 套件 | release | feat-performance-opt |\n|------|---------|---------------------|\n`; + md += `| Python pytest | ${meta.tests.release?.py?.summary ?? '—'} | ${meta.tests.feat?.py?.summary ?? '—'} |\n`; + md += `| JavaScript jest | ${meta.tests.release?.js?.summary ?? '—'} | ${meta.tests.feat?.js?.summary ?? '—'} |\n`; + md += `| 跨语言 parity | ${meta.tests.release?.parity?.summary ?? '—'} | ${meta.tests.feat?.parity?.summary ?? '—'} |\n\n`; +} + +if (meta?.shootout_glin) { + const rs = meta.shootout_glin.release; + const fs = meta.shootout_glin.feat; + md += `## 4. Shootout torture-set（glin-profanity 单库精度）\n\n`; + md += `| 指标 | release | feat-performance-opt |\n|------|---------|---------------------|\n`; + if (rs?.f1 && fs?.f1) { + md += `| Precision | ${rs.precision ?? '—'} | ${fs.precision ?? '—'} |\n`; + md += `| Recall | ${rs.recall ?? '—'} | ${fs.recall ?? '—'} |\n`; + md += `| F1 | ${rs.f1 ?? '—'} | ${fs.f1 ?? '—'} |\n`; + md += `| FPR | ${rs.fpr ?? '—'} | ${fs.fpr ?? '—'} |\n\n`; + } else { + md += `| 结果 | ${rs?.error ?? JSON.stringify(rs) ?? '—'} | ${fs?.error ?? JSON.stringify(fs) ?? '—'} |\n\n`; + } +} + +md += `## 5. 性能对比（Lite 工作负载）\n\n`; +md += `统一预热 100 次后计时；Filter 实例在稳态路径下复用。\n\n`; +md += `### 5.1 JavaScript（packages/js）\n\n`; +md += benchTable(beforeJs, afterJs); +md += `### 5.2 Python（packages/py）\n\n`; +md += benchTable(beforePy, afterPy); + +if (fullBeforeJs && fullAfterJs && fullBeforePy && fullAfterPy) { + md += `## 6. 性能对比（Full 矩阵 · 关键项）\n\n`; + const KEY = [ + 'init:shootout', + 'init:all_languages', + 'shootout/is_profane/clean_short', + 'shootout/is_profane/evasion_wordbreak', + 'shootout/is_profane/evasion_html', + 'shootout/is_profane/evasion_masked', + 'shootout/is_profane/cjk_chinese', + 'shootout_legacy/is_profane/clean_short', + 'context_aware/is_profane/context_profanity', + 'context_aware/is_profane/context_whitelist', + 'torture_set_60x_isProfane', + 'all_languages/is_profane/clean_short', + 'cached_shootout/check_profanity/clean_short_hit', + ]; + md += `### 6.1 JavaScript\n\n`; + md += keyFullRows(fullBeforeJs, fullAfterJs, KEY); + md += `### 6.2 Python\n\n`; + md += keyFullRows(fullBeforePy, fullAfterPy, KEY); +} + +md += `## 7. 结论与建议\n\n`; +md += `### 7.1 性能\n\n`; + +const scB = beforeJs.benchmarks.find((x) => x.name === 'shootout_clean'); +const scA = afterJs.benchmarks.find((x) => x.name === 'shootout_clean'); +const pyB = beforePy.benchmarks.find((x) => x.name === 'shootout_clean'); +const pyA = afterPy.benchmarks.find((x) => x.name === 'shootout_clean'); +const initB = beforeJs.benchmarks.find((x) => x.name === 'init_shootout_config'); +const initA = afterJs.benchmarks.find((x) => x.name === 'init_shootout_config'); + +if (scB && scA) { + md += `- **JS shootout_clean**：${scB.ops_per_sec.toLocaleString()} → ${scA.ops_per_sec.toLocaleString()} ops/s（${pct(scB.ops_per_sec, scA.ops_per_sec)}）\n`; +} +if (pyB && pyA) { + md += `- **Python shootout_clean**：${pyB.ops_per_sec.toLocaleString()} → ${pyA.ops_per_sec.toLocaleString()} ops/s（${pct(pyB.ops_per_sec, pyA.ops_per_sec)}）\n`; +} +if (initB && initA) { + md += `- **JS Filter 初始化（shootout 配置）**：${initB.avg_us}µs → ${initA.avg_us}µs（AC 构建开销，建议用实例池摊销）\n`; +} + +md += `\n### 7.2 精度 / 行为\n\n`; +md += `- feat-opt 在 torture-set 上目标为 **F1=100%、FPR=0%**（含 word-break / HTML / 掩码 evasion）\n`; +md += `- CJK / emoji / 重音词形等边界 case 与 release 行为差异显著，详见 parity 测试\n\n`; + +md += `### 7.3 推荐配置\n\n`; +md += `| 场景 | 建议 |\n|------|------|\n| 高 QPS API | Filter 实例池 + \`cacheResults: true\` |\n| 最高召回 | shootout 配置（aggressive leetspeak + unicode + evasion） |\n| 与 release 行为对齐调试 | \`disable_aho_corasick: true\` 走 legacy 路径 |\n| 跨语言一致性 | 跑 \`tests/cross_language_parity_test.py\` 作为发布门禁 |\n\n`; -let md = `# glin-profanity 优化前后性能对比\n\n`; -md += `_基线：release @ \`a446a8f\`（优化前） vs 当前工作区（AC + CJK + Context-aware + Evasion 归一化）_\n\n`; -md += `_环境：${beforeJs.node} / Python ${beforePy.python}，每项预热 100 次后计时_\n\n`; -md += `> **吞吐变化** 正数=更快；**延迟变化** 正数=更快（延迟降低）\n\n`; - -md = table('JavaScript（packages/js）', beforeJs, afterJs, md); -md = table('Python（packages/py）', beforePy, afterPy, md); - -md += `## Shootout 场景（torture-set，竞品对比）\n\n`; -md += `| 指标 | 优化前 | 优化后 |\n|------|--------|--------|\n`; -md += `| F1 | 80.6% | **100.0%** |\n`; -md += `| Recall | 67.4% | **100.0%** |\n`; -md += `| FPR | 0.0% | 0.0% |\n`; -md += `| Shootout ops/s（20条/轮） | ~2,533 | ~1,894 |\n\n`; - -md += `## 结论摘要\n\n`; -md += `### 运行时检测（稳态，Filter 已构造）\n\n`; -md += `- **JS 常规路径**：\`shootout_clean\` 约 **11.5k → 39.4k ops/s（+243%）**\`，得益于 Aho-Corasick 替代全量 regex 扫描\n`; -md += `- **JS torture-set 批量**：**12.9k → 25.6k ops/s（+97%）**，60 条混合用例单条延迟 **77µs → 39µs**\n`; -md += `- **Python 提升更显著**：\`shootout_clean\` **489 → 27.3k ops/s**，\`all_languages\` **242 → 58k ops/s**（AC 对多词典场景收益最大）\n`; -md += `- **Evasion 混合文本**：JS 略降约 **16%**（额外 HTML/分隔符/掩码归一化开销）；Python 仍大幅快于优化前\n\n`; -md += `### 冷启动 / 初始化\n\n`; -md += `- **AC 词典构建**：\`new Filter(shootoutConfig)\` 初始化 JS **31µs → 8.4ms**，Python **23µs → 2.3ms**\n`; -md += `- 建议生产环境使用 **Filter 实例池**（\`getPooledFilter\` / \`get_pooled_filter\`）摊销初始化成本\n\n`; -md += `### 精度 vs 性能权衡\n\n`; -md += `- Shootout 吞吐略降（~2.5k → ~1.9k ops/s），但 **F1 从 80.6% 升至 100%**，零误报保持\n`; -md += `- Context-aware insult 路径 JS 变慢（旧版 context 实现较轻量）；whitelist 路径仍更快\n\n`; -md += `### 推荐配置\n\n`; -md += `| 场景 | 建议 |\n|------|------|\n| 高 QPS API | 实例池 + \`cacheResults: true\` |\n| 最高召回 | shootout 配置（aggressive leetspeak + unicode + evasion） |\n| 低延迟单次 | 复用 Filter 实例，避免重复构造 |\n`; +md += `---\n\n`; +md += `_复现：\`python benchmarks/run-branch-comparison.py\`，结果 JSON 在 \`benchmarks/results/\`._\n`; writeFileSync(join(__dirname, 'optimization-comparison-report.md'), md, 'utf8'); console.log('Written benchmarks/optimization-comparison-report.md'); diff --git a/benchmarks/optimization-comparison-lite.mjs b/benchmarks/optimization-comparison-lite.mjs index 1641315..fd7a9ea 100644 --- a/benchmarks/optimization-comparison-lite.mjs +++ b/benchmarks/optimization-comparison-lite.mjs @@ -8,9 +8,12 @@ import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { performance } from 'node:perf_hooks'; -import { Filter } from '../packages/js/dist/index.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = process.env.GLIN_REPO_ROOT + ? join(process.env.GLIN_REPO_ROOT) + : join(__dirname, '..'); +const { Filter } = await import(join(repoRoot, 'packages/js/dist/index.js')); const label = process.argv.includes('--label') ? process.argv[process.argv.indexOf('--label') + 1] diff --git a/benchmarks/optimization-comparison-lite.py b/benchmarks/optimization-comparison-lite.py index 5dc7b69..d0a1190 100644 --- a/benchmarks/optimization-comparison-lite.py +++ b/benchmarks/optimization-comparison-lite.py @@ -9,7 +9,11 @@ from datetime import UTC, datetime from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "packages" / "py")) +REPO_ROOT = Path(__file__).resolve().parent.parent +if "GLIN_REPO_ROOT" in __import__("os").environ: + REPO_ROOT = Path(__import__("os").environ["GLIN_REPO_ROOT"]).resolve() + +sys.path.insert(0, str(REPO_ROOT / "packages" / "py")) from glin_profanity import Filter # noqa: E402 diff --git a/benchmarks/optimization-comparison-report.md b/benchmarks/optimization-comparison-report.md index dab3d14..e595a26 100644 --- a/benchmarks/optimization-comparison-report.md +++ b/benchmarks/optimization-comparison-report.md @@ -1,73 +1,123 @@ -# glin-profanity 优化前后性能对比 +# glin-profanity：release vs feat-performance-opt 完整对比报告 -_基线：release @ `a446a8f`（优化前） vs 当前工作区（AC + CJK + Context-aware + Evasion 归一化）_ +_生成时间：2026-06-30T03:35:55.619078+00:00_ + +_基线分支：`release` @ `a446a8f`_ -_环境：v25.3.0 / Python 3.13.9，每项预热 100 次后计时_ +_对比分支：`feat-performance-opt` @ `f71542c`_ -> **吞吐变化** 正数=更快；**延迟变化** 正数=更快（延迟降低） +_运行环境：Node v25.3.0 / Python 3.13.9_ -## JavaScript（packages/js） +> **吞吐变化** 正数表示 feat-opt 更快；**延迟变化** 正数表示 feat-opt 延迟更低。 -| 工作负载 | 优化前 ops/s | 优化后 ops/s | 吞吐变化 | 优化前 µs | 优化后 µs | 延迟变化 | -|---------|-------------|-------------|---------|----------|----------|----------| -| init_shootout_config | 31,641 | 119 | -99.6% | 31.6 | 8433.67 | -26588.8% | -| torture_set_60_batch | 12,980 | 25,608 | +97.3% | 77.04 | 39.05 | +49.3% | -| basic_clean | 12,335 | 52,338 | +324.3% | 81.07 | 19.11 | +76.4% | -| shootout_clean | 11,474 | 39,407 | +243.4% | 87.15 | 25.38 | +70.9% | -| shootout_evasion_mix | 10,232 | 8,586 | -16.1% | 97.73 | 116.47 | -19.2% | -| context_aware_insult | 29,744 | 10,507 | -64.7% | 33.62 | 95.17 | -183.1% | -| context_aware_whitelist | 15,247 | 26,453 | +73.5% | 65.59 | 37.8 | +42.4% | -| cjk_chinese | 16,638 | 17,711 | +6.4% | 60.1 | 56.46 | +6.1% | -| all_languages_clean | 1,124 | 24,260 | +2058.4% | 889.41 | 41.22 | +95.4% | -| shootout_legacy_clean | 12,683 | 6,461 | -49.1% | 78.85 | 154.78 | -96.3% | +## 1. 分支差异概览 -## Python（packages/py） +| 指标 | 值 | +|------|----| +| release SHA | `a446a8f` | +| feat-performance-opt SHA | `f71542c` | +| 新增 commit 数 | 8 | +| 代码变更规模 | 91 files, +8795 / -827 lines（相对 release） | -| 工作负载 | 优化前 ops/s | 优化后 ops/s | 吞吐变化 | 优化前 µs | 优化后 µs | 延迟变化 | -|---------|-------------|-------------|---------|----------|----------|----------| -| init_shootout_config | 43,799 | 426 | -99.0% | 22.83 | 2348.57 | -10187.2% | -| torture_set_60_batch | 1,464 | 30,981 | +2016.2% | 682.86 | 32.28 | +95.3% | -| basic_clean | 487 | 61,736 | +12576.8% | 2054.72 | 16.2 | +99.2% | -| shootout_clean | 489 | 27,297 | +5482.2% | 2043.97 | 36.63 | +98.2% | -| shootout_evasion_mix | 372 | 9,159 | +2362.1% | 2689.31 | 109.18 | +95.9% | -| context_aware_insult | 2,642 | 15,139 | +473.0% | 378.49 | 66.05 | +82.5% | -| context_aware_whitelist | 821 | 86,661 | +10455.5% | 1217.49 | 11.54 | +99.1% | -| cjk_chinese | 2,609 | 34,157 | +1209.2% | 383.29 | 29.28 | +92.4% | -| all_languages_clean | 242 | 58,051 | +23888.0% | 4134.8 | 17.23 | +99.6% | -| shootout_legacy_clean | 488 | 385 | -21.1% | 2051.08 | 2599.48 | -26.7% | +
feat-performance-opt 相对 release 的 commit 列表（8） -## Shootout 场景（torture-set，竞品对比） +- f71542c chore: stop tracking local CSV comparison benchmark script +- 45bf4a5 fix: prevent 5m distance tokens from leetspeak sm false positives +- 99f3a43 fix: align PY/JS profanity detection across CJK, emoji spans, and scanner edges +- f817818 fix: harden cache keys, config export, and PY/JS leetspeak parity +- df6603c fix: populate profane_words on legacy fuzzy and context-aware legacy paths +- 4936139 fix: align profane_words collection with normalized-first fallback and result parity +- a09d389 fix: harden variant mapping, context-aware parity, and filter pool exports +- 82354e2 feat: add AC fast path, evasion normalization, and context-aware parity -| 指标 | 优化前 | 优化后 | -|------|--------|--------| -| F1 | 80.6% | **100.0%** | -| Recall | 67.4% | **100.0%** | -| FPR | 0.0% | 0.0% | -| Shootout ops/s（20条/轮） | ~2,533 | ~1,894 | +
-## 结论摘要 +## 2. 核心能力变更（feat-performance-opt） -### 运行时检测（稳态，Filter 已构造） +| 能力 | release | feat-performance-opt | +|------|---------|---------------------| +| Aho-Corasick 词典快路径 | 无 | 有（`disable_aho_corasick` 可回退 legacy） | +| Evasion 归一化（HTML/掩码/分隔符） | 无 | 有 | +| Context-aware 检测 | 基础 | 与 AC 路径对齐，parity 测试覆盖 | +| CJK 边界 / 单字策略 | 较弱 | 方向 B 拉丁边界 + 无歧义单字白名单 | +| Unicode homoglyph 表 | 较小 | Py/JS 189 条完全同步 | +| Variant span 映射 | 基础 | emoji FE0F / combining class 对齐 | +| Filter 实例池 | 无 | Py/JS `get_pooled_filter` / `getPooledFilter` | +| 跨语言 parity 测试 | 无 | `tests/cross_language_parity_test.py` | -- **JS 常规路径**：`shootout_clean` 约 **11.5k → 39.4k ops/s（+243%）**`，得益于 Aho-Corasick 替代全量 regex 扫描 -- **JS torture-set 批量**：**12.9k → 25.6k ops/s（+97%）**，60 条混合用例单条延迟 **77µs → 39µs** -- **Python 提升更显著**：`shootout_clean` **489 → 27.3k ops/s**，`all_languages` **242 → 58k ops/s**（AC 对多词典场景收益最大） -- **Evasion 混合文本**：JS 略降约 **16%**（额外 HTML/分隔符/掩码归一化开销）；Python 仍大幅快于优化前 +## 3. 测试套件 -### 冷启动 / 初始化 +| 套件 | release | feat-performance-opt | +|------|---------|---------------------| +| Python pytest | ======================== 199 passed, 1 warning in 6.28s ======================== | ======================= 513 passed, 1 warning in 53.04s ======================== | +| JavaScript jest | 未单独跑 release jest | exit 0 | +| 跨语言 parity | release 无 parity 测试 | passed (exit 0) | -- **AC 词典构建**：`new Filter(shootoutConfig)` 初始化 JS **31µs → 8.4ms**，Python **23µs → 2.3ms** -- 建议生产环境使用 **Filter 实例池**（`getPooledFilter` / `get_pooled_filter`）摊销初始化成本 +## 4. Shootout torture-set（glin-profanity 单库精度） -### 精度 vs 性能权衡 +| 指标 | release | feat-performance-opt | +|------|---------|---------------------| +| Precision | 93.1% | 100.0% | +| Recall | 62.8% | 97.7% | +| F1 | 75.0% | 98.8% | +| FPR | 11.8% | 0.0% | -- Shootout 吞吐略降（~2.5k → ~1.9k ops/s），但 **F1 从 80.6% 升至 100%**，零误报保持 -- Context-aware insult 路径 JS 变慢（旧版 context 实现较轻量）；whitelist 路径仍更快 +## 5. 性能对比（Lite 工作负载） -### 推荐配置 +统一预热 100 次后计时；Filter 实例在稳态路径下复用。 + +### 5.1 JavaScript（packages/js） + +| 工作负载 | release ops/s | feat-opt ops/s | 吞吐变化 | release µs | feat-opt µs | 延迟变化 | +|---------|--------------|---------------|---------|-----------|------------|----------| +| init_shootout_config | 26,531 | 141 | -99.5% | 37.69 | 7109.93 | -18764.2% | +| torture_set_60_batch | 12,456 | 26,925 | +116.2% | 80.29 | 37.14 | +53.7% | +| basic_clean | 10,613 | 49,161 | +363.2% | 94.22 | 20.34 | +78.4% | +| shootout_clean | 10,061 | 34,002 | +238.0% | 99.39 | 29.41 | +70.4% | +| shootout_evasion_mix | 7,617 | 14,606 | +91.8% | 131.29 | 68.47 | +47.8% | +| context_aware_insult | 26,577 | 21,072 | -20.7% | 37.63 | 47.46 | -26.1% | +| context_aware_whitelist | 11,423 | 60,541 | +430.0% | 87.54 | 16.52 | +81.1% | +| cjk_chinese | 12,739 | 49,803 | +290.9% | 78.5 | 20.08 | +74.4% | +| all_languages_clean | 760 | 38,124 | +4916.3% | 1316.15 | 26.23 | +98.0% | +| shootout_legacy_clean | 9,355 | 7,101 | -24.1% | 106.9 | 140.82 | -31.7% | + +### 5.2 Python（packages/py） + +| 工作负载 | release ops/s | feat-opt ops/s | 吞吐变化 | release µs | feat-opt µs | 延迟变化 | +|---------|--------------|---------------|---------|-----------|------------|----------| +| init_shootout_config | 35,567 | 159 | -99.6% | 28.12 | 6306.1 | -22325.7% | +| torture_set_60_batch | 1,161 | 8,815 | +659.3% | 861.06 | 113.45 | +86.8% | +| basic_clean | 314 | 22,083 | +6932.8% | 3189.7 | 45.28 | +98.6% | +| shootout_clean | 353 | 9,322 | +2540.8% | 2829.93 | 107.27 | +96.2% | +| shootout_evasion_mix | 265 | 5,852 | +2108.3% | 3773.78 | 170.87 | +95.5% | +| context_aware_insult | 3,059 | 8,117 | +165.3% | 326.9 | 123.19 | +62.3% | +| context_aware_whitelist | 582 | 59,273 | +10084.4% | 1718.68 | 16.87 | +99.0% | +| cjk_chinese | 1,876 | 45,639 | +2332.8% | 533.06 | 21.91 | +95.9% | +| all_languages_clean | 94 | 39,592 | +42019.1% | 10673.37 | 25.26 | +99.8% | +| shootout_legacy_clean | 365 | 293 | -19.7% | 2742.49 | 3407.88 | -24.3% | + +## 7. 结论与建议 + +### 7.1 性能 + +- **JS shootout_clean**：10,061 → 34,002 ops/s（+238.0%） +- **Python shootout_clean**：353 → 9,322 ops/s（+2540.8%） +- **JS Filter 初始化（shootout 配置）**：37.69µs → 7109.93µs（AC 构建开销，建议用实例池摊销） + +### 7.2 精度 / 行为 + +- feat-opt 在 torture-set 上目标为 **F1=100%、FPR=0%**（含 word-break / HTML / 掩码 evasion） +- CJK / emoji / 重音词形等边界 case 与 release 行为差异显著，详见 parity 测试 + +### 7.3 推荐配置 | 场景 | 建议 | |------|------| -| 高 QPS API | 实例池 + `cacheResults: true` | +| 高 QPS API | Filter 实例池 + `cacheResults: true` | | 最高召回 | shootout 配置（aggressive leetspeak + unicode + evasion） | -| 低延迟单次 | 复用 Filter 实例，避免重复构造 | +| 与 release 行为对齐调试 | `disable_aho_corasick: true` 走 legacy 路径 | +| 跨语言一致性 | 跑 `tests/cross_language_parity_test.py` 作为发布门禁 | + +--- + +_复现：`python benchmarks/run-branch-comparison.py`，结果 JSON 在 `benchmarks/results/`._ diff --git a/benchmarks/optimization-comparison.mjs b/benchmarks/optimization-comparison.mjs index 9a2be57..ac2d874 100644 --- a/benchmarks/optimization-comparison.mjs +++ b/benchmarks/optimization-comparison.mjs @@ -10,9 +10,12 @@ import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { performance } from 'node:perf_hooks'; -import { Filter } from '../packages/js/dist/index.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = process.env.GLIN_REPO_ROOT + ? join(process.env.GLIN_REPO_ROOT) + : join(__dirname, '..'); +const { Filter } = await import(join(repoRoot, 'packages/js/dist/index.js')); const label = process.argv.includes('--label') ? process.argv[process.argv.indexOf('--label') + 1] diff --git a/benchmarks/optimization-comparison.py b/benchmarks/optimization-comparison.py index a1ede71..dffc6ee 100644 --- a/benchmarks/optimization-comparison.py +++ b/benchmarks/optimization-comparison.py @@ -9,7 +9,11 @@ from datetime import UTC, datetime from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "packages" / "py")) +REPO_ROOT = Path(__file__).resolve().parent.parent +if "GLIN_REPO_ROOT" in __import__("os").environ: + REPO_ROOT = Path(__import__("os").environ["GLIN_REPO_ROOT"]).resolve() + +sys.path.insert(0, str(REPO_ROOT / "packages" / "py")) from glin_profanity import Filter # noqa: E402 diff --git a/benchmarks/results/after-lite-js.json b/benchmarks/results/after-lite-js.json index abb3177..0cf7696 100644 --- a/benchmarks/results/after-lite-js.json +++ b/benchmarks/results/after-lite-js.json @@ -1,66 +1,66 @@ { "label": "after", "node": "v25.3.0", - "generated_at": "2026-06-19T14:34:39.427Z", + "generated_at": "2026-06-29T15:09:39.827Z", "benchmarks": [ { "name": "init_shootout_config", - "avg_us": 8433.67, - "ops_per_sec": 119, + "avg_us": 7109.93, + "ops_per_sec": 141, "iterations": 300 }, { "name": "torture_set_60_batch", - "avg_us": 39.05, - "ops_per_sec": 25608, + "avg_us": 37.14, + "ops_per_sec": 26925, "iterations": 9000 }, { "name": "basic_clean", - "avg_us": 19.11, - "ops_per_sec": 52338, + "avg_us": 20.34, + "ops_per_sec": 49161, "iterations": 20000 }, { "name": "shootout_clean", - "avg_us": 25.38, - "ops_per_sec": 39407, + "avg_us": 29.41, + "ops_per_sec": 34002, "iterations": 20000 }, { "name": "shootout_evasion_mix", - "avg_us": 116.47, - "ops_per_sec": 8586, + "avg_us": 68.47, + "ops_per_sec": 14606, "iterations": 10000 }, { "name": "context_aware_insult", - "avg_us": 95.17, - "ops_per_sec": 10507, + "avg_us": 47.46, + "ops_per_sec": 21072, "iterations": 10000 }, { "name": "context_aware_whitelist", - "avg_us": 37.8, - "ops_per_sec": 26453, + "avg_us": 16.52, + "ops_per_sec": 60541, "iterations": 10000 }, { "name": "cjk_chinese", - "avg_us": 56.46, - "ops_per_sec": 17711, + "avg_us": 20.08, + "ops_per_sec": 49803, "iterations": 10000 }, { "name": "all_languages_clean", - "avg_us": 41.22, - "ops_per_sec": 24260, + "avg_us": 26.23, + "ops_per_sec": 38124, "iterations": 3000 }, { "name": "shootout_legacy_clean", - "avg_us": 154.78, - "ops_per_sec": 6461, + "avg_us": 140.82, + "ops_per_sec": 7101, "iterations": 20000, "has_ac_fast_path": true } diff --git a/benchmarks/results/after-lite-py.json b/benchmarks/results/after-lite-py.json index 33f6f76..9c6c9d6 100644 --- a/benchmarks/results/after-lite-py.json +++ b/benchmarks/results/after-lite-py.json @@ -1,66 +1,66 @@ { "label": "after", "python": "3.13.9", - "generated_at": "2026-06-19T14:34:50.224285+00:00", + "generated_at": "2026-06-29T15:08:21.355104+00:00", "benchmarks": [ { "name": "init_shootout_config", - "avg_us": 2348.57, - "ops_per_sec": 426, + "avg_us": 6306.1, + "ops_per_sec": 159, "iterations": 300 }, { "name": "torture_set_60_batch", - "avg_us": 32.28, - "ops_per_sec": 30981, + "avg_us": 113.45, + "ops_per_sec": 8815, "iterations": 9000 }, { "name": "basic_clean", - "avg_us": 16.2, - "ops_per_sec": 61736, + "avg_us": 45.28, + "ops_per_sec": 22083, "iterations": 20000 }, { "name": "shootout_clean", - "avg_us": 36.63, - "ops_per_sec": 27297, + "avg_us": 107.27, + "ops_per_sec": 9322, "iterations": 20000 }, { "name": "shootout_evasion_mix", - "avg_us": 109.18, - "ops_per_sec": 9159, + "avg_us": 170.87, + "ops_per_sec": 5852, "iterations": 10000 }, { "name": "context_aware_insult", - "avg_us": 66.05, - "ops_per_sec": 15139, + "avg_us": 123.19, + "ops_per_sec": 8117, "iterations": 10000 }, { "name": "context_aware_whitelist", - "avg_us": 11.54, - "ops_per_sec": 86661, + "avg_us": 16.87, + "ops_per_sec": 59273, "iterations": 10000 }, { "name": "cjk_chinese", - "avg_us": 29.28, - "ops_per_sec": 34157, + "avg_us": 21.91, + "ops_per_sec": 45639, "iterations": 10000 }, { "name": "all_languages_clean", - "avg_us": 17.23, - "ops_per_sec": 58051, + "avg_us": 25.26, + "ops_per_sec": 39592, "iterations": 3000 }, { "name": "shootout_legacy_clean", - "avg_us": 2599.48, - "ops_per_sec": 385, + "avg_us": 3407.88, + "ops_per_sec": 293, "iterations": 20000, "has_ac_fast_path": true } diff --git a/benchmarks/results/before-lite-js.json b/benchmarks/results/before-lite-js.json index e052ad5..328e9d3 100644 --- a/benchmarks/results/before-lite-js.json +++ b/benchmarks/results/before-lite-js.json @@ -1,66 +1,66 @@ { "label": "before", "node": "v25.3.0", - "generated_at": "2026-06-19T14:38:32.408Z", + "generated_at": "2026-06-29T15:14:29.493Z", "benchmarks": [ { "name": "init_shootout_config", - "avg_us": 31.6, - "ops_per_sec": 31641, + "avg_us": 37.69, + "ops_per_sec": 26531, "iterations": 300 }, { "name": "torture_set_60_batch", - "avg_us": 77.04, - "ops_per_sec": 12980, + "avg_us": 80.29, + "ops_per_sec": 12456, "iterations": 9000 }, { "name": "basic_clean", - "avg_us": 81.07, - "ops_per_sec": 12335, + "avg_us": 94.22, + "ops_per_sec": 10613, "iterations": 20000 }, { "name": "shootout_clean", - "avg_us": 87.15, - "ops_per_sec": 11474, + "avg_us": 99.39, + "ops_per_sec": 10061, "iterations": 20000 }, { "name": "shootout_evasion_mix", - "avg_us": 97.73, - "ops_per_sec": 10232, + "avg_us": 131.29, + "ops_per_sec": 7617, "iterations": 10000 }, { "name": "context_aware_insult", - "avg_us": 33.62, - "ops_per_sec": 29744, + "avg_us": 37.63, + "ops_per_sec": 26577, "iterations": 10000 }, { "name": "context_aware_whitelist", - "avg_us": 65.59, - "ops_per_sec": 15247, + "avg_us": 87.54, + "ops_per_sec": 11423, "iterations": 10000 }, { "name": "cjk_chinese", - "avg_us": 60.1, - "ops_per_sec": 16638, + "avg_us": 78.5, + "ops_per_sec": 12739, "iterations": 10000 }, { "name": "all_languages_clean", - "avg_us": 889.41, - "ops_per_sec": 1124, + "avg_us": 1316.15, + "ops_per_sec": 760, "iterations": 3000 }, { "name": "shootout_legacy_clean", - "avg_us": 78.85, - "ops_per_sec": 12683, + "avg_us": 106.9, + "ops_per_sec": 9355, "iterations": 20000, "has_ac_fast_path": true } diff --git a/benchmarks/results/before-lite-py.json b/benchmarks/results/before-lite-py.json index 9c7aad2..7bcf08b 100644 --- a/benchmarks/results/before-lite-py.json +++ b/benchmarks/results/before-lite-py.json @@ -1,66 +1,66 @@ { "label": "before", "python": "3.13.9", - "generated_at": "2026-06-19T14:38:43.614663+00:00", + "generated_at": "2026-06-29T15:09:48.152791+00:00", "benchmarks": [ { "name": "init_shootout_config", - "avg_us": 22.83, - "ops_per_sec": 43799, + "avg_us": 28.12, + "ops_per_sec": 35567, "iterations": 300 }, { "name": "torture_set_60_batch", - "avg_us": 682.86, - "ops_per_sec": 1464, + "avg_us": 861.06, + "ops_per_sec": 1161, "iterations": 9000 }, { "name": "basic_clean", - "avg_us": 2054.72, - "ops_per_sec": 487, + "avg_us": 3189.7, + "ops_per_sec": 314, "iterations": 20000 }, { "name": "shootout_clean", - "avg_us": 2043.97, - "ops_per_sec": 489, + "avg_us": 2829.93, + "ops_per_sec": 353, "iterations": 20000 }, { "name": "shootout_evasion_mix", - "avg_us": 2689.31, - "ops_per_sec": 372, + "avg_us": 3773.78, + "ops_per_sec": 265, "iterations": 10000 }, { "name": "context_aware_insult", - "avg_us": 378.49, - "ops_per_sec": 2642, + "avg_us": 326.9, + "ops_per_sec": 3059, "iterations": 10000 }, { "name": "context_aware_whitelist", - "avg_us": 1217.49, - "ops_per_sec": 821, + "avg_us": 1718.68, + "ops_per_sec": 582, "iterations": 10000 }, { "name": "cjk_chinese", - "avg_us": 383.29, - "ops_per_sec": 2609, + "avg_us": 533.06, + "ops_per_sec": 1876, "iterations": 10000 }, { "name": "all_languages_clean", - "avg_us": 4134.8, - "ops_per_sec": 242, + "avg_us": 10673.37, + "ops_per_sec": 94, "iterations": 3000 }, { "name": "shootout_legacy_clean", - "avg_us": 2051.08, - "ops_per_sec": 488, + "avg_us": 2742.49, + "ops_per_sec": 365, "iterations": 20000, "has_ac_fast_path": true } diff --git a/benchmarks/results/branch-comparison.json b/benchmarks/results/branch-comparison.json new file mode 100644 index 0000000..c2ac7d0 --- /dev/null +++ b/benchmarks/results/branch-comparison.json @@ -0,0 +1,340 @@ +{ + "generated_at": "2026-06-30T03:35:55.619078+00:00", + "release_ref": "release", + "feat_ref": "feat-performance-opt", + "release_sha": "a446a8f", + "feat_sha": "f71542c", + "commits": [ + "f71542c chore: stop tracking local CSV comparison benchmark script", + "45bf4a5 fix: prevent 5m distance tokens from leetspeak sm false positives", + "99f3a43 fix: align PY/JS profanity detection across CJK, emoji spans, and scanner edges", + "f817818 fix: harden cache keys, config export, and PY/JS leetspeak parity", + "df6603c fix: populate profane_words on legacy fuzzy and context-aware legacy paths", + "4936139 fix: align profane_words collection with normalized-first fallback and result parity", + "a09d389 fix: harden variant mapping, context-aware parity, and filter pool exports", + "82354e2 feat: add AC fast path, evasion normalization, and context-aware parity" + ], + "benchmarks": { + "lite": { + "before_py": { + "label": "before", + "python": "3.13.9", + "generated_at": "2026-06-29T15:09:48.152791+00:00", + "benchmarks": [ + { + "name": "init_shootout_config", + "avg_us": 28.12, + "ops_per_sec": 35567, + "iterations": 300 + }, + { + "name": "torture_set_60_batch", + "avg_us": 861.06, + "ops_per_sec": 1161, + "iterations": 9000 + }, + { + "name": "basic_clean", + "avg_us": 3189.7, + "ops_per_sec": 314, + "iterations": 20000 + }, + { + "name": "shootout_clean", + "avg_us": 2829.93, + "ops_per_sec": 353, + "iterations": 20000 + }, + { + "name": "shootout_evasion_mix", + "avg_us": 3773.78, + "ops_per_sec": 265, + "iterations": 10000 + }, + { + "name": "context_aware_insult", + "avg_us": 326.9, + "ops_per_sec": 3059, + "iterations": 10000 + }, + { + "name": "context_aware_whitelist", + "avg_us": 1718.68, + "ops_per_sec": 582, + "iterations": 10000 + }, + { + "name": "cjk_chinese", + "avg_us": 533.06, + "ops_per_sec": 1876, + "iterations": 10000 + }, + { + "name": "all_languages_clean", + "avg_us": 10673.37, + "ops_per_sec": 94, + "iterations": 3000 + }, + { + "name": "shootout_legacy_clean", + "avg_us": 2742.49, + "ops_per_sec": 365, + "iterations": 20000, + "has_ac_fast_path": true + } + ] + }, + "after_py": { + "label": "after", + "python": "3.13.9", + "generated_at": "2026-06-29T15:08:21.355104+00:00", + "benchmarks": [ + { + "name": "init_shootout_config", + "avg_us": 6306.1, + "ops_per_sec": 159, + "iterations": 300 + }, + { + "name": "torture_set_60_batch", + "avg_us": 113.45, + "ops_per_sec": 8815, + "iterations": 9000 + }, + { + "name": "basic_clean", + "avg_us": 45.28, + "ops_per_sec": 22083, + "iterations": 20000 + }, + { + "name": "shootout_clean", + "avg_us": 107.27, + "ops_per_sec": 9322, + "iterations": 20000 + }, + { + "name": "shootout_evasion_mix", + "avg_us": 170.87, + "ops_per_sec": 5852, + "iterations": 10000 + }, + { + "name": "context_aware_insult", + "avg_us": 123.19, + "ops_per_sec": 8117, + "iterations": 10000 + }, + { + "name": "context_aware_whitelist", + "avg_us": 16.87, + "ops_per_sec": 59273, + "iterations": 10000 + }, + { + "name": "cjk_chinese", + "avg_us": 21.91, + "ops_per_sec": 45639, + "iterations": 10000 + }, + { + "name": "all_languages_clean", + "avg_us": 25.26, + "ops_per_sec": 39592, + "iterations": 3000 + }, + { + "name": "shootout_legacy_clean", + "avg_us": 3407.88, + "ops_per_sec": 293, + "iterations": 20000, + "has_ac_fast_path": true + } + ] + }, + "before_js": { + "label": "before", + "node": "v25.3.0", + "generated_at": "2026-06-29T15:14:29.493Z", + "benchmarks": [ + { + "name": "init_shootout_config", + "avg_us": 37.69, + "ops_per_sec": 26531, + "iterations": 300 + }, + { + "name": "torture_set_60_batch", + "avg_us": 80.29, + "ops_per_sec": 12456, + "iterations": 9000 + }, + { + "name": "basic_clean", + "avg_us": 94.22, + "ops_per_sec": 10613, + "iterations": 20000 + }, + { + "name": "shootout_clean", + "avg_us": 99.39, + "ops_per_sec": 10061, + "iterations": 20000 + }, + { + "name": "shootout_evasion_mix", + "avg_us": 131.29, + "ops_per_sec": 7617, + "iterations": 10000 + }, + { + "name": "context_aware_insult", + "avg_us": 37.63, + "ops_per_sec": 26577, + "iterations": 10000 + }, + { + "name": "context_aware_whitelist", + "avg_us": 87.54, + "ops_per_sec": 11423, + "iterations": 10000 + }, + { + "name": "cjk_chinese", + "avg_us": 78.5, + "ops_per_sec": 12739, + "iterations": 10000 + }, + { + "name": "all_languages_clean", + "avg_us": 1316.15, + "ops_per_sec": 760, + "iterations": 3000 + }, + { + "name": "shootout_legacy_clean", + "avg_us": 106.9, + "ops_per_sec": 9355, + "iterations": 20000, + "has_ac_fast_path": true + } + ] + }, + "after_js": { + "label": "after", + "node": "v25.3.0", + "generated_at": "2026-06-29T15:09:39.827Z", + "benchmarks": [ + { + "name": "init_shootout_config", + "avg_us": 7109.93, + "ops_per_sec": 141, + "iterations": 300 + }, + { + "name": "torture_set_60_batch", + "avg_us": 37.14, + "ops_per_sec": 26925, + "iterations": 9000 + }, + { + "name": "basic_clean", + "avg_us": 20.34, + "ops_per_sec": 49161, + "iterations": 20000 + }, + { + "name": "shootout_clean", + "avg_us": 29.41, + "ops_per_sec": 34002, + "iterations": 20000 + }, + { + "name": "shootout_evasion_mix", + "avg_us": 68.47, + "ops_per_sec": 14606, + "iterations": 10000 + }, + { + "name": "context_aware_insult", + "avg_us": 47.46, + "ops_per_sec": 21072, + "iterations": 10000 + }, + { + "name": "context_aware_whitelist", + "avg_us": 16.52, + "ops_per_sec": 60541, + "iterations": 10000 + }, + { + "name": "cjk_chinese", + "avg_us": 20.08, + "ops_per_sec": 49803, + "iterations": 10000 + }, + { + "name": "all_languages_clean", + "avg_us": 26.23, + "ops_per_sec": 38124, + "iterations": 3000 + }, + { + "name": "shootout_legacy_clean", + "avg_us": 140.82, + "ops_per_sec": 7101, + "iterations": 20000, + "has_ac_fast_path": true + } + ] + } + }, + "full": null + }, + "tests": { + "release": { + "py": { + "summary": "======================== 199 passed, 1 warning in 6.28s ========================" + }, + "js": { + "summary": "未单独跑 release jest" + }, + "parity": { + "summary": "release 无 parity 测试" + } + }, + "feat": { + "py": { + "summary": "======================= 513 passed, 1 warning in 53.04s ========================" + }, + "js": { + "summary": "exit 0" + }, + "parity": { + "summary": "passed (exit 0)" + } + } + }, + "shootout_glin": { + "release": { + "precision": "93.1%", + "recall": "62.8%", + "f1": "75.0%", + "fpr": "11.8%", + "tp": 27, + "fp": 2, + "tn": 15, + "fn": 16 + }, + "feat": { + "precision": "100.0%", + "recall": "97.7%", + "f1": "98.8%", + "fpr": "0.0%", + "tp": 42, + "fp": 0, + "tn": 17, + "fn": 1 + } + } +} diff --git a/benchmarks/run-branch-comparison.py b/benchmarks/run-branch-comparison.py new file mode 100755 index 0000000..0a42b1c --- /dev/null +++ b/benchmarks/run-branch-comparison.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Run release vs feat-performance-opt benchmarks (JS + PY) and emit JSON results.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from datetime import UTC, datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BENCH = ROOT / "benchmarks" +RESULTS = BENCH / "results" +RELEASE_REF = "release" +FEAT_REF = "feat-performance-opt" +RUN_FULL = "--full" in sys.argv + + +def run( + cmd: list[str], + *, + cwd: Path | None = None, + env: dict | None = None, + check: bool = True, +) -> subprocess.CompletedProcess: + merged = os.environ.copy() + if env: + merged.update(env) + print(f"$ {' '.join(cmd)}", flush=True) + return subprocess.run( + cmd, + cwd=cwd or ROOT, + env=merged, + check=check, + capture_output=True, + text=True, + ) + + +def git_rev(ref: str) -> str: + return run(["git", "rev-parse", "--short", ref]).stdout.strip() + + +def git_log_range(base: str, head: str) -> list[str]: + out = run(["git", "log", f"{base}..{head}", "--oneline"]).stdout.strip() + return [line for line in out.splitlines() if line.strip()] + + +def prepare_worktree(ref: str) -> Path: + worktrees_root = ROOT / ".benchmark-worktrees" + worktrees_root.mkdir(exist_ok=True) + path = worktrees_root / ref.replace("/", "_") + if path.exists(): + run(["git", "worktree", "remove", "--force", str(path)], check=False) + if path.exists(): + shutil.rmtree(path) + run(["git", "worktree", "add", "--detach", str(path), ref]) + return path + + +def build_js(tree: Path) -> None: + js_dir = tree / "packages" / "js" + if not (js_dir / "node_modules").exists(): + run(["npm", "install"], cwd=js_dir, env={"CI": "true"}) + run(["npm", "run", "build"], cwd=js_dir) + + +def run_py_benchmark(script: Path, label: str, tree: Path) -> dict: + env = { + "GLIN_REPO_ROOT": str(tree.resolve()), + "PYTHONDONTWRITEBYTECODE": "1", + } + proc = run( + [sys.executable, str(script), "--label", label], + cwd=ROOT, + env=env, + ) + return json.loads(proc.stdout) + + +def run_js_benchmark(script: Path, label: str, tree: Path) -> dict: + proc = run( + ["node", str(script), "--label", label], + cwd=ROOT, + env={"GLIN_REPO_ROOT": str(tree.resolve())}, + ) + return json.loads(proc.stdout) + + +def run_glin_shootout_metrics(tree: Path) -> dict: + torture_path = ROOT / "benchmarks" / "shootout" / "torture-set.json" + cases = json.loads(torture_path.read_text(encoding="utf-8")) + code = """ +import json, os, sys +from pathlib import Path +root = Path(os.environ["GLIN_REPO_ROOT"]) +sys.path.insert(0, str(root / "packages" / "py")) +from glin_profanity import Filter + +cases = json.loads(Path(os.environ["TORTURE_PATH"]).read_text(encoding="utf-8")) +f = Filter({ + "languages": ["english"], + "detect_leetspeak": True, + "leetspeak_level": "aggressive", + "normalize_unicode": True, +}) +tp = fp = tn = fn = 0 +for case in cases: + flagged = f.is_profane(case["input"]) + expect = case.get("shouldFlag", case.get("expect") == "profane") + if flagged and expect: tp += 1 + elif flagged and not expect: fp += 1 + elif not flagged and expect: fn += 1 + else: tn += 1 +precision = tp / (tp + fp) if (tp + fp) else 0 +recall = tp / (tp + fn) if (tp + fn) else 0 +f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0 +fpr = fp / (fp + tn) if (fp + tn) else 0 +print(json.dumps({ + "precision": f"{precision*100:.1f}%", + "recall": f"{recall*100:.1f}%", + "f1": f"{f1*100:.1f}%", + "fpr": f"{fpr*100:.1f}%", + "tp": tp, "fp": fp, "tn": tn, "fn": fn, +})) +""" + proc = run( + [sys.executable, "-c", code], + cwd=ROOT, + env={ + "GLIN_REPO_ROOT": str(tree.resolve()), + "TORTURE_PATH": str(torture_path), + }, + ) + return json.loads(proc.stdout.strip()) + + +def run_tests( + tree: Path, + ref: str, + *, + js: bool = True, + parity: bool = True, +) -> dict: + summary: dict = {"ref": ref, "js": {}, "py": {}, "parity": {}} + + py_proc = run( + [sys.executable, "-m", "pytest", "-q", "--tb=no"], + cwd=tree / "packages" / "py", + env={"GLIN_REPO_ROOT": str(tree.resolve())}, + ) + last = py_proc.stdout.strip().splitlines()[-1] if py_proc.stdout.strip() else "" + summary["py"]["summary"] = last + + js_dir = tree / "packages" / "js" + if js and (js_dir / "node_modules").exists(): + js_proc = run(["npm", "test", "--", "--ci", "--watchAll=false"], cwd=js_dir) + tail = js_proc.stdout.strip().splitlines() + summary["js"]["summary"] = next( + (line for line in reversed(tail) if "Tests:" in line or "passed" in line), + tail[-1] if tail else "", + ) + + parity_path = tree / "tests" / "cross_language_parity_test.py" + if parity and parity_path.exists(): + try: + parity_proc = run([sys.executable, str(parity_path)], cwd=tree) + lines = parity_proc.stdout.strip().splitlines() + summary["parity"]["summary"] = lines[-1] if lines else "passed" + except subprocess.CalledProcessError as exc: + summary["parity"]["summary"] = (exc.stdout or exc.stderr or str(exc)).strip().splitlines()[-1] + else: + summary["parity"]["summary"] = "not present on this ref" + + return summary + + +def run_shootout_glin(tree: Path) -> dict: + try: + return run_glin_shootout_metrics(tree) + except subprocess.CalledProcessError as exc: + return {"error": (exc.stderr or exc.stdout or str(exc))[-500:]} + + +def main() -> None: + RESULTS.mkdir(parents=True, exist_ok=True) + release_sha = git_rev(RELEASE_REF) + feat_sha = git_rev(FEAT_REF) + commits = git_log_range(RELEASE_REF, FEAT_REF) + + print("=== Building feat-performance-opt JS ===") + build_js(ROOT) + + print("=== Running feat-performance-opt benchmarks ===") + after_lite_py = run_py_benchmark(BENCH / "optimization-comparison-lite.py", "after", ROOT) + after_lite_js = run_js_benchmark(BENCH / "optimization-comparison-lite.mjs", "after", ROOT) + after_full_py = after_full_js = None + if RUN_FULL: + after_full_py = run_py_benchmark(BENCH / "optimization-comparison.py", "after", ROOT) + after_full_js = run_js_benchmark(BENCH / "optimization-comparison.mjs", "after", ROOT) + + print("=== Preparing release worktree ===") + release_tree = prepare_worktree(RELEASE_REF) + build_js(release_tree) + + print("=== Running release benchmarks ===") + before_lite_py = run_py_benchmark(BENCH / "optimization-comparison-lite.py", "before", release_tree) + before_lite_js = run_js_benchmark(BENCH / "optimization-comparison-lite.mjs", "before", release_tree) + before_full_py = before_full_js = None + if RUN_FULL: + before_full_py = run_py_benchmark(BENCH / "optimization-comparison.py", "before", release_tree) + before_full_js = run_js_benchmark(BENCH / "optimization-comparison.mjs", "before", release_tree) + + print("=== Running test suites (feat only; release pytest only) ===") + release_tests = run_tests(release_tree, RELEASE_REF, js=False, parity=False) + feat_tests = run_tests(ROOT, FEAT_REF, js=True, parity=True) + + print("=== Running shootout (glin only) ===") + release_shootout = run_shootout_glin(release_tree) + feat_shootout = run_shootout_glin(ROOT) + + payload = { + "generated_at": datetime.now(UTC).isoformat(), + "release_ref": RELEASE_REF, + "feat_ref": FEAT_REF, + "release_sha": release_sha, + "feat_sha": feat_sha, + "commits": commits, + "benchmarks": { + "lite": { + "before_py": before_lite_py, + "after_py": after_lite_py, + "before_js": before_lite_js, + "after_js": after_lite_js, + }, + "full": { + "before_py": before_full_py, + "after_py": after_full_py, + "before_js": before_full_js, + "after_js": after_full_js, + }, + }, + "tests": {"release": release_tests, "feat": feat_tests}, + "shootout_glin": {"release": release_shootout, "feat": feat_shootout}, + } + + names = { + "before-lite-py.json": before_lite_py, + "after-lite-py.json": after_lite_py, + "before-lite-js.json": before_lite_js, + "after-lite-js.json": after_lite_js, + "branch-comparison.json": payload, + } + if before_full_py and after_full_py: + names["before-py.json"] = before_full_py + names["after-py.json"] = after_full_py + if before_full_js and after_full_js: + names["before-js.json"] = before_full_js + names["after-js.json"] = after_full_js + for name, data in names.items(): + (RESULTS / name).write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + print("=== Generating markdown report ===") + run(["node", str(BENCH / "generate-comparison-report.mjs")]) + + print(f"\nDone. Report: {BENCH / 'optimization-comparison-report.md'}") + + +if __name__ == "__main__": + main() From c1d309d8afa0cc2e5e4ea3e998dd06e56ff3f2ff Mon Sep 17 00:00:00 2001 From: wlike Date: Tue, 30 Jun 2026 14:11:53 +0800 Subject: [PATCH 10/11] fix: guard HTML entity decode against out-of-range codepoints Reject invalid numeric entities before chr/fromCodePoint and preserve them through html.unescape so malformed input cannot crash evasion normalization. Co-authored-by: Cursor --- packages/js/src/utils/evasion.ts | 28 ++++++++++-- packages/js/tests/leetspeak-unicode.test.ts | 7 +++ packages/py/glin_profanity/utils/evasion.py | 47 ++++++++++++++++----- packages/py/tests/test_evasion.py | 10 +++++ 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/packages/js/src/utils/evasion.ts b/packages/js/src/utils/evasion.ts index dcda9a4..6bd35bd 100644 --- a/packages/js/src/utils/evasion.ts +++ b/packages/js/src/utils/evasion.ts @@ -4,20 +4,40 @@ * @module utils/evasion */ +const MAX_UNICODE_CODEPOINT = 0x10ffff; + +function isValidUnicodeScalar(code: number): boolean { + if (!Number.isFinite(code) || code < 0 || code > MAX_UNICODE_CODEPOINT) { + return false; + } + return code < 0xd800 || code > 0xdfff; +} + +function safeCodePoint(code: number, fallback: string): string { + if (!isValidUnicodeScalar(code)) { + return fallback; + } + try { + return String.fromCodePoint(code); + } catch { + return fallback; + } +} + /** * Removes HTML tags and decodes common numeric/named entities. */ export function stripHtmlAndDecodeEntities(text: string): string { let result = text.replace(/<[^>]*>/g, ''); - result = result.replace(/&#(\d+);/g, (_, dec: string) => { + result = result.replace(/&#(\d+);/g, (entity, dec: string) => { const code = parseInt(dec, 10); - return Number.isFinite(code) ? String.fromCodePoint(code) : _; + return safeCodePoint(code, entity); }); - result = result.replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) => { + result = result.replace(/&#x([0-9a-fA-F]+);/g, (entity, hex: string) => { const code = parseInt(hex, 16); - return Number.isFinite(code) ? String.fromCodePoint(code) : _; + return safeCodePoint(code, entity); }); return result diff --git a/packages/js/tests/leetspeak-unicode.test.ts b/packages/js/tests/leetspeak-unicode.test.ts index ec9fa60..fd923ff 100644 --- a/packages/js/tests/leetspeak-unicode.test.ts +++ b/packages/js/tests/leetspeak-unicode.test.ts @@ -30,6 +30,13 @@ describe('Evasion Normalization', () => { expect(stripHtmlAndDecodeEntities('😀')).toBe('😀'); expect(stripHtmlAndDecodeEntities('😀')).toBe('😀'); }); + + it('should leave invalid numeric entities unchanged', () => { + expect(stripHtmlAndDecodeEntities('�')).toBe('�'); + expect(stripHtmlAndDecodeEntities('�')).toBe('�'); + expect(stripHtmlAndDecodeEntities('�')).toBe('�'); + expect(normalizeEvasion('hello � world')).toBe('hello � world'); + }); }); describe('collapseSeparatedCharacters', () => { diff --git a/packages/py/glin_profanity/utils/evasion.py b/packages/py/glin_profanity/utils/evasion.py index 5f72fa5..85939af 100644 --- a/packages/py/glin_profanity/utils/evasion.py +++ b/packages/py/glin_profanity/utils/evasion.py @@ -10,6 +10,7 @@ ) _NUMERIC_ENTITY_PATTERN = re.compile(r"&#(\d+);") _HEX_ENTITY_PATTERN = re.compile(r"&#x([0-9a-fA-F]+);") +_REMAINING_NUMERIC_ENTITY_PATTERN = re.compile(r"&#(?:\d+|x[0-9a-fA-F]+);") _MASKED_PATTERNS: list[tuple[re.Pattern[str], str]] = [ (re.compile(r"\bf\*+cking\b", re.IGNORECASE), "fucking"), (re.compile(r"\bf\*+ck\b", re.IGNORECASE), "fuck"), @@ -17,22 +18,48 @@ (re.compile(r"\bf\*{2,}(?=\W|$)", re.IGNORECASE), "fuck"), (re.compile(r"\bf\s+yourself\b", re.IGNORECASE), "fuck yourself"), ] +_MAX_UNICODE_CODEPOINT = 0x10FFFF + + +def _is_valid_unicode_scalar(code: int) -> bool: + if not 0 <= code <= _MAX_UNICODE_CODEPOINT: + return False + return not 0xD800 <= code <= 0xDFFF + + +def _safe_codepoint_char(match: re.Match[str], base: int) -> str: + """Decode a numeric entity, leaving invalid/out-of-range values unchanged.""" + try: + code = int(match.group(1), base) + except ValueError: + return match.group(0) + if not _is_valid_unicode_scalar(code): + return match.group(0) + return chr(code) + + +def _escape_remaining_numeric_entities(text: str) -> str: + """Prevent html.unescape from re-processing invalid numeric entities.""" + + def escape(match: re.Match[str]) -> str: + return match.group(0).replace("&", "&", 1) + + return _REMAINING_NUMERIC_ENTITY_PATTERN.sub(escape, text) def strip_html_and_decode_entities(text: str) -> str: """Remove HTML tags and decode numeric/named entities.""" result = re.sub(r"<[^>]*>", "", text) - def decode_numeric(match: re.Match[str]) -> str: - code = int(match.group(1)) - return chr(code) - - def decode_hex(match: re.Match[str]) -> str: - code = int(match.group(1), 16) - return chr(code) - - result = _NUMERIC_ENTITY_PATTERN.sub(decode_numeric, result) - result = _HEX_ENTITY_PATTERN.sub(decode_hex, result) + result = _NUMERIC_ENTITY_PATTERN.sub( + lambda match: _safe_codepoint_char(match, 10), + result, + ) + result = _HEX_ENTITY_PATTERN.sub( + lambda match: _safe_codepoint_char(match, 16), + result, + ) + result = _escape_remaining_numeric_entities(result) return html.unescape(result) diff --git a/packages/py/tests/test_evasion.py b/packages/py/tests/test_evasion.py index 945e031..a9f3dc9 100644 --- a/packages/py/tests/test_evasion.py +++ b/packages/py/tests/test_evasion.py @@ -18,6 +18,16 @@ def test_strip_html_and_decode_entities(self) -> None: assert strip_html_and_decode_entities("😀") == "😀" assert strip_html_and_decode_entities("😀") == "😀" + def test_invalid_numeric_entities_are_left_unchanged(self) -> None: + assert strip_html_and_decode_entities("�") == "�" + assert strip_html_and_decode_entities("�") == "�" + assert strip_html_and_decode_entities("�") == "�" + assert normalize_evasion("hello � world") == "hello � world" + + def test_malformed_entities_do_not_crash_filter(self) -> None: + assert strip_html_and_decode_entities("�") == "�" + assert not Filter({"languages": ["english"]}).is_profane("hello �") + def test_collapse_separated_characters(self) -> None: assert collapse_separated_characters("f.u.c.k") == "fuck" assert collapse_separated_characters("f_u_c_k") == "fuck" From a5a3bdd1643fffd0ee99878e734dbe5415a2309c Mon Sep 17 00:00:00 2001 From: wlike Date: Wed, 1 Jul 2026 12:36:55 +0800 Subject: [PATCH 11/11] fix: decode named entities before tag strip and accept uppercase hex entities Decode </> before removing HTML tags to close entity-encoded tag evasion, and match &#X...; hex entities with safe bounds checking in PY/JS. Co-authored-by: Cursor --- packages/js/src/utils/evasion.ts | 10 +++++----- packages/js/tests/leetspeak-unicode.test.ts | 4 ++++ packages/py/glin_profanity/utils/evasion.py | 11 +++++------ packages/py/tests/test_evasion.py | 4 ++++ 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/js/src/utils/evasion.ts b/packages/js/src/utils/evasion.ts index 6bd35bd..c2b4b6e 100644 --- a/packages/js/src/utils/evasion.ts +++ b/packages/js/src/utils/evasion.ts @@ -28,24 +28,24 @@ function safeCodePoint(code: number, fallback: string): string { * Removes HTML tags and decodes common numeric/named entities. */ export function stripHtmlAndDecodeEntities(text: string): string { - let result = text.replace(/<[^>]*>/g, ''); - - result = result.replace(/&#(\d+);/g, (entity, dec: string) => { + let result = text.replace(/&#(\d+);/g, (entity, dec: string) => { const code = parseInt(dec, 10); return safeCodePoint(code, entity); }); - result = result.replace(/&#x([0-9a-fA-F]+);/g, (entity, hex: string) => { + result = result.replace(/&#[xX]([0-9a-fA-F]+);/g, (entity, hex: string) => { const code = parseInt(hex, 16); return safeCodePoint(code, entity); }); - return result + result = result .replace(/</gi, '<') .replace(/>/gi, '>') .replace(/&/gi, '&') .replace(/"/gi, '"') .replace(/'/gi, "'"); + + return result.replace(/<[^>]*>/g, ''); } /** diff --git a/packages/js/tests/leetspeak-unicode.test.ts b/packages/js/tests/leetspeak-unicode.test.ts index fd923ff..f8427ab 100644 --- a/packages/js/tests/leetspeak-unicode.test.ts +++ b/packages/js/tests/leetspeak-unicode.test.ts @@ -29,12 +29,16 @@ describe('Evasion Normalization', () => { expect(stripHtmlAndDecodeEntities('a
ss')).toBe('ass'); expect(stripHtmlAndDecodeEntities('😀')).toBe('😀'); expect(stripHtmlAndDecodeEntities('😀')).toBe('😀'); + expect(stripHtmlAndDecodeEntities('😀')).toBe('😀'); + expect(stripHtmlAndDecodeEntities('f<b>u</b>ck')).toBe('fuck'); }); it('should leave invalid numeric entities unchanged', () => { expect(stripHtmlAndDecodeEntities('�')).toBe('�'); expect(stripHtmlAndDecodeEntities('�')).toBe('�'); + expect(stripHtmlAndDecodeEntities('�')).toBe('�'); expect(stripHtmlAndDecodeEntities('�')).toBe('�'); + expect(stripHtmlAndDecodeEntities('�')).toBe('�'); expect(normalizeEvasion('hello � world')).toBe('hello � world'); }); }); diff --git a/packages/py/glin_profanity/utils/evasion.py b/packages/py/glin_profanity/utils/evasion.py index 85939af..c3190bf 100644 --- a/packages/py/glin_profanity/utils/evasion.py +++ b/packages/py/glin_profanity/utils/evasion.py @@ -9,8 +9,8 @@ r"\b([a-zA-Z0-9@$!#*])(?:[\s._\-]+([a-zA-Z0-9@$!#*])){2,}\b" ) _NUMERIC_ENTITY_PATTERN = re.compile(r"&#(\d+);") -_HEX_ENTITY_PATTERN = re.compile(r"&#x([0-9a-fA-F]+);") -_REMAINING_NUMERIC_ENTITY_PATTERN = re.compile(r"&#(?:\d+|x[0-9a-fA-F]+);") +_HEX_ENTITY_PATTERN = re.compile(r"&#[xX]([0-9a-fA-F]+);") +_REMAINING_NUMERIC_ENTITY_PATTERN = re.compile(r"&#(?:\d+|[xX][0-9a-fA-F]+);") _MASKED_PATTERNS: list[tuple[re.Pattern[str], str]] = [ (re.compile(r"\bf\*+cking\b", re.IGNORECASE), "fucking"), (re.compile(r"\bf\*+ck\b", re.IGNORECASE), "fuck"), @@ -49,18 +49,17 @@ def escape(match: re.Match[str]) -> str: def strip_html_and_decode_entities(text: str) -> str: """Remove HTML tags and decode numeric/named entities.""" - result = re.sub(r"<[^>]*>", "", text) - result = _NUMERIC_ENTITY_PATTERN.sub( lambda match: _safe_codepoint_char(match, 10), - result, + text, ) result = _HEX_ENTITY_PATTERN.sub( lambda match: _safe_codepoint_char(match, 16), result, ) result = _escape_remaining_numeric_entities(result) - return html.unescape(result) + result = html.unescape(result) + return re.sub(r"<[^>]*>", "", result) def collapse_separated_characters(text: str) -> str: diff --git a/packages/py/tests/test_evasion.py b/packages/py/tests/test_evasion.py index a9f3dc9..9fe1b09 100644 --- a/packages/py/tests/test_evasion.py +++ b/packages/py/tests/test_evasion.py @@ -17,11 +17,15 @@ def test_strip_html_and_decode_entities(self) -> None: # Astral code points (Python chr handles these natively). assert strip_html_and_decode_entities("😀") == "😀" assert strip_html_and_decode_entities("😀") == "😀" + assert strip_html_and_decode_entities("😀") == "😀" + assert strip_html_and_decode_entities("f<b>u</b>ck") == "fuck" def test_invalid_numeric_entities_are_left_unchanged(self) -> None: assert strip_html_and_decode_entities("�") == "�" assert strip_html_and_decode_entities("�") == "�" + assert strip_html_and_decode_entities("�") == "�" assert strip_html_and_decode_entities("�") == "�" + assert strip_html_and_decode_entities("�") == "�" assert normalize_evasion("hello � world") == "hello � world" def test_malformed_entities_do_not_crash_filter(self) -> None: