Skip to content

Commit 8620705

Browse files
kckernclaude
kckern
and
claude
committed
fix(web): stem fallback search with Sphinx's stem_en, not Porter (1980)
The stack workflow's Sphinx-vs-fallback check found Sphinx matching godly, earthly and heavenly to god, earth and heaven where the fallback's Porter stemmer gave godli, earthli and heavenli. Sphinx's stem_en (sphinxstemen.cpp) is Porter2 with its own exceptions: Step 2 deletes -li after c d e g h k m n r t, y after a vowel stays (days -> day), only -> onli, was -> wa, dying -> die, gener- keeps its R1. search/stemEn.js ports it line by line; compiled and run side by side, the C code and the port agree on all 288,696 ASCII words of the full database and a dictionary, where Porter (1980) differed on 9,645 of them. stemPrefix drops the e that stem_en adds to vowel-consonant stems (aging -> age, using -> use) and maps die/lie/tie/sky to d/l/t/sk, so every word in the database still starts with its stem's prefix. The comparison script now also asks for godly, earthly things, holy, only and daily. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 7204599 commit 8620705

8 files changed

Lines changed: 293 additions & 118 deletions

File tree

‎apps/web/src/services/search/matcher.js‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
// Emulates how the Sphinx index (infra/sphinx/sphinx.conf) tokenizes and matches verse text:
22
// charset_table = 0..9, A..Z->a..z, _, a..z and Hangul; everything else separates words;
3-
// morphology = stem_en with min_stemming_len = 3. A plain query matches when every query
3+
// morphology = stem_en with min_stemming_len = 3 (stemEn.js). A plain query matches when every query
44
// word's stem occurs in the verse (Sphinx's implicit AND); a "quoted phrase" matches its words
55
// in order and adjacent (Sphinx phrase operator), which the live site passed through unchanged.
6-
import { stem } from './porterStemmer.js';
6+
import { stem } from './stemEn.js';
77

88
const TOKEN = /[0-9a-z_가-힣ᄀ-ᅙᅡ-ᆢᆨ-ᇹ]+/g;
99

@@ -62,8 +62,14 @@ function termPrefixes(terms) {
6262
}
6363

6464
// The longest prefix every word with this stem shares, for a coarse SQL prefilter.
65-
// Porter may rewrite the stem's last letter (happy → happi, day → dai), so drop it when the stem
66-
// is long enough or ends in that i.
65+
// stem_en may rewrite the stem's last letter (happy → happi, day → dai) or add an e (aging → age,
66+
// using → use), so drop the last letter when the stem is long enough, ends in that i, or is
67+
// vowel-consonant-e; dying, lying and tying stem to die, lie and tie, and skies to sky.
68+
// (Checked against every word of the full database: each starts with its stem's prefix.)
69+
const RESPELLED = new Map([['die', 'd'], ['lie', 'l'], ['tie', 't'], ['sky', 'sk']]);
6770
export function stemPrefix(s) {
68-
return s.length > 3 || (s.length > 1 && s.endsWith('i')) ? s.slice(0, -1) : s;
71+
if (RESPELLED.has(s)) return RESPELLED.get(s);
72+
if (s.length > 3 || (s.length > 1 && s.endsWith('i'))) return s.slice(0, -1);
73+
if (s.length === 3 && /^[aeiou][^aeiou]e$/.test(s)) return s.slice(0, -1);
74+
return s;
6975
}

‎apps/web/src/services/search/porterStemmer.js‎

Lines changed: 0 additions & 101 deletions
This file was deleted.

‎apps/web/src/services/search/stemEn.js‎

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
// Sphinx's English stemmer (`morphology = stem_en`), ported line by line from src/sphinxstemen.cpp
2+
// of github.com/sphinxsearch/sphinx (2.x; the 3.x source is not published, and the stack workflow
3+
// compares the fallback with 3.x), built without SNOWBALL2011. The port gives the same stem as
4+
// the compiled C code for every ASCII word of the full database and a 235,000-word dictionary.
5+
// It is Porter's algorithm in the Snowball "Porter2" form with Sphinx's own exceptions, and
6+
// differs from Porter (1980) in places, e.g. godly → god, earthly → earth, but holy → holi,
7+
// family → famili; days → day; was → wa, dying → die.
8+
// The MySQL search fallback stems with it so its matches follow the Sphinx index.
9+
// Input is a lowercase ASCII word ([0-9a-z_]); other characters count as consonants.
10+
11+
const VOWELS = new Set(['a', 'e', 'i', 'o', 'u', 'y']);
12+
const DOUBLES = 'bdfgmnprt';
13+
// The letters before a final -li that Step 2 deletes it after (Porter2's valid li-endings)
14+
const LI_ENDINGS = 'bcdeghkmnrt';
15+
16+
// 3-letter shortcuts left as they are, and two the C code cuts to two letters
17+
const KEEP3 = new Set(['the', 'and', 'you', 'for', 'her', 'she', 'but', 'had', 'sky']);
18+
const CUT3 = new Set(['was', 'his']);
19+
const KEEP4 = new Set(['that', 'with', 'said', 'from', 'news', 'howe']);
20+
const EXCEPTIONS = new Map([
21+
['proceed', 'proceed'], ['exceed', 'exceed'], ['succeed', 'succeed'],
22+
['dying', 'die'], ['lying', 'lie'], ['tying', 'tie'],
23+
['inning', 'inning'], ['outing', 'outing'], ['canning', 'canning'],
24+
['skies', 'sky'], ['innings', 'inning'], ['outings', 'outing'], ['cannings', 'canning'],
25+
['idly', 'idl'], ['gently', 'gentl'], ['ugly', 'ugli'], ['early', 'earli'], ['only', 'onli'], ['singly', 'singl'],
26+
]);
27+
28+
export function stem(input) {
29+
let len = input.length;
30+
if (len <= 2) return input;
31+
if (len === 3) {
32+
if (KEEP3.has(input)) return input;
33+
if (CUT3.has(input)) return input.slice(0, 2);
34+
}
35+
if (len === 4 && KEEP4.has(input)) return input;
36+
if (len >= 4 && len <= 8 && EXCEPTIONS.has(input)) return EXCEPTIONS.get(input);
37+
38+
const word = input.split('');
39+
const isVowel = i => VOWELS.has(word[i]); // 'Y' (a hidden consonant y) is not a vowel
40+
const ivwxy = c => VOWELS.has(c) || c === 'w' || c === 'x' || c === 'Y';
41+
const W = (p, c) => word[len - p] === c;
42+
// the word ends in `s`
43+
const suff = s => len >= s.length && [...s].every((c, k) => W(s.length - k, c));
44+
// the word ends in `s` followed by any one letter (the C code's SUFFnA)
45+
const suffA = s => len >= s.length + 1 && [...s].every((c, k) => W(s.length + 1 - k, c));
46+
47+
// hide consonant-style y's
48+
let hasY = false;
49+
if (word[0] === 'y') { word[0] = 'Y'; hasY = true; }
50+
for (let i = 1; i < len; i++) if (word[i] === 'y' && isVowel(i - 1)) { word[i] = 'Y'; hasY = true; }
51+
52+
// regions: R1 after the first vowel-consonant pair, R2 after the second
53+
let r1, firstVowel, i;
54+
if (len >= 5 && input.startsWith('gener')) {
55+
r1 = 5;
56+
firstVowel = 1;
57+
} else {
58+
for (i = 0; i < len && !isVowel(i); i++);
59+
firstVowel = i;
60+
for (i = firstVowel; i < len - 2; i++) if (isVowel(i) && !isVowel(i + 1)) break;
61+
r1 = i + 2;
62+
}
63+
for (i = r1; i < len - 2; i++) if (isVowel(i) && !isVowel(i + 1)) break;
64+
const r2 = i + 2;
65+
66+
// Step 1a
67+
const ied = () => { if (len-- !== 4) len--; };
68+
switch (word[len - 1]) {
69+
case 'd':
70+
if (word[len - 3] === 'i' && word[len - 2] === 'e') ied();
71+
break;
72+
case 's':
73+
if (suff('sses')) len -= 2;
74+
else if (word[len - 3] === 'i' && word[len - 2] === 'e') ied();
75+
else if (word[len - 2] !== 'u' && word[len - 2] !== 's') len--;
76+
break;
77+
}
78+
79+
// Step 1b
80+
let cut = 0;
81+
switch (word[len - 1]) {
82+
case 'd':
83+
if (suffA('ee')) { if (len - 3 >= r1) len--; break; }
84+
if (word[len - 2] === 'e') cut = 2;
85+
break;
86+
case 'y':
87+
if (word[len - 2] === 'l') {
88+
if (suffA('eedl')) { if (len - 5 >= r1) len -= 3; break; }
89+
if (suffA('edl')) { cut = 4; break; }
90+
if (suffA('ingl')) { cut = 5; break; }
91+
}
92+
break;
93+
case 'g':
94+
if (suffA('in')) cut = 3;
95+
break;
96+
}
97+
if (cut && firstVowel < len - cut) {
98+
len -= cut;
99+
if (suff('at') || suff('bl') || suff('iz')) word[len++] = 'e';
100+
else if (len >= 2 && word[len - 1] === word[len - 2] && DOUBLES.includes(word[len - 1])) len--;
101+
else if ((len === 2 && isVowel(0) && !isVowel(1))
102+
|| (len === r1 && !isVowel(len - 3) && isVowel(len - 2) && !ivwxy(word[len - 1]))) {
103+
word[len++] = 'e';
104+
}
105+
}
106+
107+
// Step 1c
108+
if (len > 2 && (word[len - 1] === 'y' || word[len - 1] === 'Y') && !isVowel(len - 2)) word[len - 1] = 'i';
109+
110+
// Step 2
111+
if (len - 2 >= r1) {
112+
switch (word[len - 1]) {
113+
case 'i': {
114+
if (len >= 3 && (W(2, 'c') || W(2, 'l') || W(2, 't'))) {
115+
let done = true;
116+
if (suffA('enc') || suffA('anc') || suffA('abl')) { if (len - 4 >= r1) word[len - 1] = 'e'; }
117+
else if (suffA('bl')) { if (len - 3 >= r1) word[len - 1] = 'e'; }
118+
else if (suffA('entl')) { if (len - 5 >= r1) len -= 2; }
119+
else if (suffA('alit')) { if (len - 5 >= r1) len -= 3; }
120+
else if (suffA('ousl')) { if (len - 5 >= r1) len -= 2; }
121+
else if (suffA('ivit')) { if (len - 5 >= r1) { word[len - 3] = 'e'; len -= 2; } }
122+
else if (suffA('bilit')) { if (len - 6 >= r1) { word[len - 5] = 'l'; word[len - 4] = 'e'; len -= 3; } }
123+
else if (suffA('full')) { if (len - 5 >= r1) len -= 2; }
124+
else if (suffA('lessl')) { if (len - 6 >= r1) len -= 2; }
125+
else done = false;
126+
if (done) break;
127+
}
128+
if (len - 3 >= r1 && suffA('og')) { len -= 1; break; }
129+
if (len - 2 >= r1 && word[len - 2] === 'l') len -= 2;
130+
else break;
131+
if (len - 2 >= r1 && suff('al')) {
132+
len -= 2;
133+
if (len - 5 >= r1 && suff('ation')) { len -= 3; word[len++] = 'e'; break; }
134+
if (suff('tion')) break;
135+
len += 2;
136+
} else if (!LI_ENDINGS.includes(word[len - 1])) {
137+
len += 2;
138+
}
139+
break;
140+
}
141+
case 'l':
142+
if (suffA('ationa')) { if (len - 7 >= r1) { word[len - 5] = 'e'; len -= 4; } break; }
143+
if (suffA('tiona')) { if (len - 6 >= r1) len -= 2; break; }
144+
break;
145+
case 'm':
146+
if (suffA('alis')) { if (len - 5 >= r1) len -= 3; }
147+
break;
148+
case 'n':
149+
if (suffA('izatio')) { if (len - 7 >= r1) { word[len - 5] = 'e'; len -= 4; } break; }
150+
if (suffA('atio')) { if (len - 5 >= r1) { word[len - 3] = 'e'; len -= 2; } break; }
151+
break;
152+
case 'r':
153+
if (suffA('ize')) { if (len - 4 >= r1) len -= 1; break; }
154+
if (suffA('ato')) { if (len - 4 >= r1) { word[len - 2] = 'e'; len -= 1; } break; }
155+
break;
156+
case 's':
157+
if (len - 7 >= r1 && (suffA('fulnes') || suffA('ousnes') || suffA('ivenes'))) len -= 4;
158+
break;
159+
}
160+
}
161+
162+
// Step 3
163+
if (len - 3 >= r1) {
164+
switch (word[len - 1]) {
165+
case 'e':
166+
if (suffA('aliz')) { if (len - 5 >= r1) len -= 3; break; }
167+
if (suffA('icat')) { if (len - 5 >= r1) len -= 3; break; }
168+
if (suffA('ativ')) { if (len - 5 >= r1) len -= 5; break; }
169+
break;
170+
case 'i':
171+
if (suffA('icit')) { if (len - 5 >= r1) len -= 3; }
172+
break;
173+
case 'l':
174+
if (suffA('ica')) { if (len - 4 >= r1) len -= 2; break; }
175+
if (suffA('fu')) { len -= 3; break; }
176+
break;
177+
case 's':
178+
if (suffA('nes')) { if (len - 4 >= r1) len -= 4; }
179+
break;
180+
}
181+
}
182+
183+
// Step 4
184+
if (len - 2 >= r2) {
185+
switch (word[len - 1]) {
186+
case 'c':
187+
if (word[len - 2] === 'i') len -= 2;
188+
break;
189+
case 'e':
190+
if (len - 3 >= r2) {
191+
if (suffA('anc') || suffA('enc') || suffA('abl') || suffA('ibl')) { if (len - 4 >= r2) len -= 4; break; }
192+
if (suffA('at') || suffA('iv') || suffA('iz')) { len -= 3; break; }
193+
}
194+
break;
195+
case 'i':
196+
if (suffA('it')) { if (len - 3 >= r2) len -= 3; }
197+
break;
198+
case 'l':
199+
if (word[len - 2] === 'a') len -= 2;
200+
break;
201+
case 'm':
202+
if (suffA('is')) { if (len - 3 >= r2) len -= 3; }
203+
break;
204+
case 'n':
205+
if (len - 3 >= r2 && suff('ion') && (word[len - 4] === 't' || word[len - 4] === 's')) len -= 3;
206+
break;
207+
case 'r':
208+
if (word[len - 2] === 'e') len -= 2;
209+
break;
210+
case 's':
211+
if (suffA('ou')) { if (len - 3 >= r2) len -= 3; }
212+
break;
213+
case 't':
214+
if (word[len - 2] === 'n') {
215+
if (suffA('emen')) { if (len - 5 >= r2) len -= 5; break; }
216+
if (suffA('men')) { if (len - 4 >= r2) len -= 4; break; }
217+
if (suffA('an')) { if (len - 3 >= r2) len -= 3; break; }
218+
if (suffA('en')) { if (len - 3 >= r2) len -= 3; break; }
219+
}
220+
break;
221+
}
222+
}
223+
224+
// Step 5
225+
if (word[len - 1] === 'e') {
226+
if (len > r2) len--;
227+
else if (len <= r1) { /* keep */ }
228+
else if (len > 3 && !isVowel(len - 4) && isVowel(len - 3) && !ivwxy(word[len - 2])) { /* keep */ }
229+
else if (len === 3 && isVowel(0) && !isVowel(1)) { /* keep */ }
230+
else len--;
231+
}
232+
if (len > r2 && word[len - 1] === 'l' && word[len - 2] === 'l') len--;
233+
234+
const out = word.slice(0, len).join('');
235+
return hasY ? out.replace(/Y/g, 'y') : out;
236+
}
237+
238+
export default stem;

‎apps/web/test/integration/sphinx-vs-fallback.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import { diffSets, hasDifference, isPhrase, formatQuery, shownPairs } from './sp
2121
const QUERIES = [
2222
'loved', 'love', 'loving', 'charity', 'faith hope charity', 'in the beginning', 'beginning',
2323
'water', 'waters', 'lord god', "lord's", 'days', 'begat', 'unto', 'went up', 'jesus wept',
24-
'heaven and earth', 'children of israel',
24+
'heaven and earth', 'children of israel', 'godly', 'earthly things', 'holy', 'only', 'daily',
2525
'"in the beginning"', '"faith hope"', '"the lord"',
2626
];
2727
const SPHINX_DEFAULT_MAX_MATCHES = 1000;

‎apps/web/test/integration/sphinxDiff.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Comparison and report logic for sphinx-vs-fallback.mjs, kept free of I/O for unit tests.
22
import { queryTerms, matchesTerms, tokenize, stemPrefix } from '../../src/services/search/matcher.js';
3-
import { stem } from '../../src/services/search/porterStemmer.js';
3+
import { stem } from '../../src/services/search/stemEn.js';
44
import { plainText } from '../../src/domain/rules/legacyText.js';
55

66
export const MAX_SHOWN = 10;

‎apps/web/test/integration/sphinxDiff.test.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ describe('diagnose', () => {
5656

5757
describe('stemmedWords and snippet', () => {
5858
test('words that could carry a query stem, with their stems', () => {
59-
expect(stemmedWords('Beloved, let us love one another; lovers of days', 'loving days')).toBe('love, lovers→lover, days→dai');
59+
expect(stemmedWords('Beloved, let us love one another; lovers of days', 'loving days')).toBe('love, lovers→lover, days→day');
6060
});
6161
test('short text is shown whole', () => {
6262
expect(snippet('Jesus wept.', 'wept')).toBe('Jesus wept.');

0 commit comments

Comments
 (0)