Skip to content

Commit 76d2932

Browse files
TMHSDigitalclaude
andcommitted
Search reads entry bodies (Notes and Evidence) too (#92)
The build writes search-bodies.json (each listed entry's body as plain text, URLs and Markdown syntax dropped; 182 KB gzipped). The catalog fetches it on the first search, never on page load, and rebuilds its index with a fourth field weighted below names, tags and summaries, so current rankings hold. Tested with a body-only word; checked in a browser (a word found only in one entry's Notes returns it; no fetch before searching). No snippet on the card yet. Closes #92 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LpkeXQPaUhxjgqdqnk6BrJ
1 parent a59100d commit 76d2932

5 files changed

Lines changed: 60 additions & 4 deletions

File tree

‎ROADMAP.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ missing entry, so freshness and accuracy come before volume.
2121

2222
## Planned
2323

24-
- Full-text search over entry bodies (Notes and Evidence), not only names, tags and summaries.
2524

2625
## Done
2726

27+
- Full-text search over entry bodies (Notes and Evidence): `search-bodies.json`, fetched on the first search, ranked below name, tag and summary hits.
2828
- A changelog page: [docs/review-ledger.md](docs/review-ledger.md) on the site as guide/changelog/, and the README "Recent" block written from it by `npm run counts`.
2929
- Site search across names, tags and summaries, with licence, format and category filters.
3030
- The guides in docs/ rendered as site pages, and an engine filter that hides projects,

‎site/build.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import fs from "node:fs";
1515
import path from "node:path";
1616
import { fileURLToPath } from "node:url";
1717
import { baseHeadHtml, BROWSER_MODULES, isListed, commercialLabel, headAssetsHtml, esc, latestAllowedDate, OG_CARD_NAME, opensearchLinkHtml, opensearchXml, PAGE_MODULES, siteFooterHtml, siteHeaderHtml } from "./lib/shared.mjs";
18+
import { bodySearchText } from "./lib/search.mjs";
1819
import { filmstripHtml, groupedEntriesHtml } from "./lib/cards.mjs";
1920
import { guidePageHtml } from "./lib/guide-page.mjs";
2021
import { renderGuide } from "./lib/guide-render.mjs";
@@ -602,6 +603,9 @@ function main() {
602603
writeOut("data.json", JSON.stringify(payload, null, 2));
603604
copyOut(path.join(__dirname, "data.schema.json"), "data.schema.json");
604605
writeOut("data.js", `window.__CATALOG__ = ${JSON.stringify(payload)};\n`);
606+
// Entry bodies for full-text search, fetched on the first search, not on
607+
// page load (#92). Listed entries only, as search shows.
608+
writeOut("search-bodies.json", JSON.stringify(Object.fromEntries(entries.filter(isListed).map((e) => [e.id, bodySearchText(bodies.get(e.id) || "")]))));
605609
writeOut("catalog.csv", catalogCsv(entries, config.site.siteUrl));
606610
writeOut("feed.xml", atomFeed({ entries, site: config.site, generatedAt: payload.generatedAt, kind: "added" }));
607611
writeOut("reverified.xml", atomFeed({ entries, site: config.site, generatedAt: payload.generatedAt, kind: "verified" }));

‎site/lib/lib.test.mjs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2436,6 +2436,19 @@ await section("review ledger: changelog and README Recent block (#91)", async ()
24362436
eq("recent: no sections, no block", recentBlock([]), "");
24372437
});
24382438

2439+
/* full-text search over entry bodies (#92) -------------------------------- */
2440+
await section("full-text search over entry bodies (#92)", async () => {
2441+
const { searchFields, scoreFields, queryTests, bodySearchText } = await import("./search.mjs");
2442+
const e = { name: "Kit", tags: ["ui"], license: "CC0", category: "2d", summary: "A small kit." };
2443+
const body = bodySearchText("## Notes\n\n- Ships a **zorbulator** sheet, see [the page](https://x.test/zorb).\n\n## Evidence\n\n- Page (2026-09-01): \"CC0\"");
2444+
lacks("body text: URLs are dropped", body, "https://");
2445+
has("body text: link labels and words kept", body, "the page");
2446+
eq("body: a word only in the Notes finds the entry once bodies are loaded", scoreFields(searchFields(e, body), queryTests("zorbulator")) > 0, true);
2447+
eq("body: without the bodies it does not", scoreFields(searchFields(e), queryTests("zorbulator")), 0);
2448+
eq("body: a body hit ranks below a summary hit", scoreFields(searchFields(e, body), queryTests("zorbulator")) < scoreFields(searchFields(e, body), queryTests("small")), true);
2449+
eq("body: a name hit still ranks first", scoreFields(searchFields(e, body), queryTests("kit")), 3);
2450+
});
2451+
24392452
/* report (keep last) ------------------------------------------------------ */
24402453
if (failures.length) {
24412454
console.error(`lib.test failed (${failures.length}):`);

‎site/lib/search.mjs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,35 @@ export function joinedCompounds(text) {
3131
* metadata, which ranks above a summary-only hit. Tags no longer repeat the
3232
* publisher or licence (V19, V21), so those fields are searched directly.
3333
*/
34-
export const FIELD_WEIGHTS = [3, 2, 1];
35-
export function searchFields(e) {
34+
export const FIELD_WEIGHTS = [3, 2, 1, 0.5];
35+
/**
36+
* `body` is the entry's Notes and Evidence as plain text (search-bodies.json,
37+
* loaded on the first search; #92). It is the last and lightest field, so a
38+
* body-only hit ranks below every metadata or summary hit.
39+
*/
40+
export function searchFields(e, body = "") {
3641
const field = (text) => normalize(`${text} ${joinedCompounds(text)}`);
3742
return [
3843
field([e.name, e.publisher || "", ...(e.tags || [])].join(" ")),
3944
field([e.license, e.category, PERSPECTIVE_SEARCH[e.camera_perspective] || "", ...(e.formats || []), ...(e.subcategories || [])].join(" ")),
4045
field(e.summary || ""),
46+
body ? normalize(body) : "",
4147
];
4248
}
4349

50+
/**
51+
* An entry body as searchable text: Markdown link and emphasis syntax and
52+
* URLs dropped, headings kept as words. Built once per entry by build.mjs.
53+
*/
54+
export function bodySearchText(md) {
55+
return String(md)
56+
.replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1")
57+
.replace(/https?:\/\/\S+/g, " ")
58+
.replace(/[#*_`>]/g, " ")
59+
.replace(/\s+/g, " ")
60+
.trim();
61+
}
62+
4463
/**
4564
* A few words readers use for the same thing as the catalog's own wording.
4665
* A query word also matches each listed phrase (normalised text, so a phrase

‎site/public/app.js‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,13 +127,33 @@ import { isListed, PERSPECTIVES } from "./shared.mjs";
127127

128128
// Each entry's searchable fields, normalised once, on the first search (search.js).
129129
let searchIndex = null;
130+
// Entry bodies (Notes and Evidence) arrive on the first search (#92); the
131+
// index is rebuilt once with them and the view re-filtered.
132+
let bodies = null;
133+
let bodiesRequested = false;
134+
function loadBodies() {
135+
if (bodiesRequested) return;
136+
bodiesRequested = true;
137+
fetch("search-bodies.json")
138+
.then((r) => (r.ok ? r.json() : null))
139+
.then((map) => {
140+
if (!map) return;
141+
bodies = map;
142+
searchIndex = null;
143+
if (state.q.trim()) apply();
144+
})
145+
.catch(() => {
146+
// Offline or blocked: search keeps to names, tags and summaries.
147+
});
148+
}
130149
function fieldsOf(entry) {
131-
if (!searchIndex) searchIndex = new Map(data.entries.map((e) => [e.id, searchFields(e)]));
150+
if (!searchIndex) searchIndex = new Map(data.entries.map((e) => [e.id, searchFields(e, bodies?.[e.id] || "")]));
132151
return searchIndex.get(entry.id) || [];
133152
}
134153

135154
let compiled = { q: null, tests: [] };
136155
function queryTests() {
156+
if (state.q.trim()) loadBodies();
137157
if (compiled.q !== state.q) compiled = { q: state.q, tests: compileQuery(state.q) };
138158
return compiled.tests;
139159
}

0 commit comments

Comments
 (0)