Skip to content

Commit f06878b

Browse files
authored
Merge pull request #505 from VirtualFlyBrain/feature/term-sitemaps
Add /reports/<id> viewer URLs to the term sitemap alongside static pages
2 parents 22701cd + 9cc8b75 commit f06878b

1 file changed

Lines changed: 109 additions & 0 deletions

File tree

‎vfbterms.py‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,99 @@ def get_report_url(identifier):
201201
"""Create the VFB report URL path for a term identifier."""
202202
return f'/reports/{identifier}'
203203

204+
# ─── Sitemap Generation ─────────────────────────────────────────────────────
205+
#
206+
# Two URLs exist per term -- the static description page this script writes
207+
# (/term/<slug>/, Hugo's own canonical for that page) and the interactive
208+
# viewer (/reports/<id>, which 302s to the Geppetto viewer and is what the
209+
# viewer itself declares as canonical -- see geppetto-vfb's pageMetadata.js).
210+
# Both are real, separately useful pages, so both belong in the sitemap.
211+
#
212+
# Term pages carry `sitemap_exclude: true` (see generate_page below) because
213+
# Hugo's own sitemap.xml has no pagination: it would emit one file with the
214+
# whole multi-hundred-thousand-term corpus in it. This writes the sharded
215+
# sitemap1.xml.. Google expects instead, plus the sitemap-terms.xml index
216+
# that robots.txt already advertises.
217+
SITEMAP_SITE_BASE = "https://www.virtualflybrain.org"
218+
SITEMAP_REPORT_BASE = "https://virtualflybrain.org"
219+
# Google caps a sitemap file at 50,000 URLs; each term contributes two.
220+
SITEMAP_MAX_TERMS_PER_SHARD = 25000
221+
222+
_sitemap_entries = {}
223+
_sitemap_lock = threading.Lock()
224+
225+
_CANONICAL_SLUG_RE = re.compile(r'canonicalUrl:\s*"https://www\.virtualflybrain\.org/term/([^/]+)/"')
226+
227+
228+
def record_sitemap_entry(term_id, url_slug):
229+
"""Remember one term's slug for the end-of-run sitemap write. Thread-safe."""
230+
if not term_id or not url_slug:
231+
return
232+
with _sitemap_lock:
233+
_sitemap_entries[term_id] = url_slug
234+
235+
236+
def slug_from_existing_page(filename):
237+
"""Recover a term's URL slug from a page already on disk (the "skip" path
238+
in process_term, which never re-fetches term_data, so never recomputes the
239+
slug the normal way)."""
240+
try:
241+
with open(filename, "r", encoding="utf-8") as f:
242+
for _ in range(20):
243+
line = f.readline()
244+
if not line:
245+
break
246+
match = _CANONICAL_SLUG_RE.search(line)
247+
if match:
248+
return match.group(1)
249+
except OSError:
250+
pass
251+
return None
252+
253+
254+
def write_sitemaps(output_dir):
255+
"""Write sitemap1.xml.. (both the /term/ page and the /reports/ viewer
256+
URL per term) and the sitemap-terms.xml index over them.
257+
258+
Call once, after every group has been processed, so the corpus is
259+
complete -- process_group() covers the full current ID list from Neo4j
260+
on every run, "skip" included, so by then _sitemap_entries holds every
261+
term regardless of whether this run touched its page.
262+
"""
263+
if not output_dir:
264+
return
265+
with _sitemap_lock:
266+
entries = sorted(_sitemap_entries.items())
267+
if not entries:
268+
print("WARNING: no sitemap entries collected -- not writing sitemaps")
269+
return
270+
271+
os.makedirs(output_dir, exist_ok=True)
272+
shard_names = []
273+
for shard_start in range(0, len(entries), SITEMAP_MAX_TERMS_PER_SHARD):
274+
shard = entries[shard_start:shard_start + SITEMAP_MAX_TERMS_PER_SHARD]
275+
shard_index = len(shard_names) + 1
276+
shard_name = f"sitemap{shard_index}.xml"
277+
lines = ['<?xml version="1.0" encoding="UTF-8"?>',
278+
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
279+
for term_id, url_slug in shard:
280+
lines.append(f' <url><loc>{SITEMAP_SITE_BASE}/term/{url_slug}/</loc></url>')
281+
lines.append(f' <url><loc>{SITEMAP_REPORT_BASE}/reports/{term_id}</loc></url>')
282+
lines.append('</urlset>')
283+
with open(join(output_dir, shard_name), "w", encoding="utf-8") as f:
284+
f.write("\n".join(lines) + "\n")
285+
shard_names.append(shard_name)
286+
287+
index_lines = ['<?xml version="1.0" encoding="UTF-8"?>',
288+
'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
289+
for shard_name in shard_names:
290+
index_lines.append(f' <sitemap><loc>{SITEMAP_SITE_BASE}/{shard_name}</loc></sitemap>')
291+
index_lines.append('</sitemapindex>')
292+
with open(join(output_dir, "sitemap-terms.xml"), "w", encoding="utf-8") as f:
293+
f.write("\n".join(index_lines) + "\n")
294+
295+
print(f"Wrote {len(shard_names)} sitemap shard(s) covering {len(entries)} terms to {output_dir}")
296+
204297
def get_query_results_url(term_id, query_name):
205298
"""Create a VFB viewer URL for a term query result set."""
206299
if not query_name:
@@ -860,13 +953,17 @@ def process_term(term_id):
860953
"""
861954
filename = term_id + "_v" + str(version) + ".md"
862955
if os.path.isfile(filename):
956+
# Unchanged term: no re-fetch, so recover the slug already on disk
957+
# rather than skip the sitemap entry entirely.
958+
record_sitemap_entry(term_id, slug_from_existing_page(filename))
863959
return "skip"
864960

865961
term_data = fetch_term_info(term_id)
866962
if term_data is None:
867963
return "fail"
868964

869965
page_content = generate_page(term_data)
966+
record_sitemap_entry(term_id, get_term_url(term_data.get("Name", ""), term_id))
870967

871968
tmp = f"{filename}.{os.getpid()}.{threading.get_ident()}.tmp"
872969
with open(tmp, "w", encoding="utf-8") as f:
@@ -1145,6 +1242,13 @@ def test_medulla_page():
11451242
if __name__ == "__main__":
11461243
if len(sys.argv) > 1:
11471244
mypath = sys.argv[1]
1245+
# Optional: where to write sitemap1.xml.. and sitemap-terms.xml once
1246+
# the full corpus below has been processed. Point this at wherever
1247+
# is served as static files at the site root (e.g. the Hugo `public`
1248+
# directory the deploy already publishes from) -- robots.txt already
1249+
# advertises /sitemap-terms.xml, so dropping the files there is all
1250+
# that is needed for them to take effect.
1251+
sitemap_dir = sys.argv[2] if len(sys.argv) > 2 else None
11481252
print("Updating all files in " + mypath)
11491253
groups = [
11501254
('fbbt/', 'FBbt classes', "MATCH (n:Class) WHERE n.short_form starts with 'FBbt' WITH n.short_form as id ORDER BY id ASC RETURN collect(distinct id) as ids"),
@@ -1174,6 +1278,11 @@ def test_medulla_page():
11741278
for relative_dir, label, query in groups:
11751279
process_group(mypath, relative_dir, label, query)
11761280

1281+
if sitemap_dir:
1282+
write_sitemaps(sitemap_dir)
1283+
else:
1284+
print("No sitemap output dir given (3rd argument) -- skipping sitemap write.")
1285+
11771286
else:
11781287
print("Testing term page generation...")
11791288
success = test_medulla_page()

0 commit comments

Comments
 (0)