@@ -201,6 +201,99 @@ def get_report_url(identifier):
201201 """Create the VFB report URL path for a term identifier."""
202202 return f'/reports/{ identifier } '
203203
204+ # ─── Sitemap Generation ─────────────────────────────────────────────────────
205+ #
206+ # Two URLs exist per term -- the static description page this script writes
207+ # (/term/<slug>/, Hugo's own canonical for that page) and the interactive
208+ # viewer (/reports/<id>, which 302s to the Geppetto viewer and is what the
209+ # viewer itself declares as canonical -- see geppetto-vfb's pageMetadata.js).
210+ # Both are real, separately useful pages, so both belong in the sitemap.
211+ #
212+ # Term pages carry `sitemap_exclude: true` (see generate_page below) because
213+ # Hugo's own sitemap.xml has no pagination: it would emit one file with the
214+ # whole multi-hundred-thousand-term corpus in it. This writes the sharded
215+ # sitemap1.xml.. Google expects instead, plus the sitemap-terms.xml index
216+ # that robots.txt already advertises.
217+ SITEMAP_SITE_BASE = "https://www.virtualflybrain.org"
218+ SITEMAP_REPORT_BASE = "https://virtualflybrain.org"
219+ # Google caps a sitemap file at 50,000 URLs; each term contributes two.
220+ SITEMAP_MAX_TERMS_PER_SHARD = 25000
221+
222+ _sitemap_entries = {}
223+ _sitemap_lock = threading .Lock ()
224+
225+ _CANONICAL_SLUG_RE = re .compile (r'canonicalUrl:\s*"https://www\.virtualflybrain\.org/term/([^/]+)/"' )
226+
227+
228+ def record_sitemap_entry (term_id , url_slug ):
229+ """Remember one term's slug for the end-of-run sitemap write. Thread-safe."""
230+ if not term_id or not url_slug :
231+ return
232+ with _sitemap_lock :
233+ _sitemap_entries [term_id ] = url_slug
234+
235+
236+ def slug_from_existing_page (filename ):
237+ """Recover a term's URL slug from a page already on disk (the "skip" path
238+ in process_term, which never re-fetches term_data, so never recomputes the
239+ slug the normal way)."""
240+ try :
241+ with open (filename , "r" , encoding = "utf-8" ) as f :
242+ for _ in range (20 ):
243+ line = f .readline ()
244+ if not line :
245+ break
246+ match = _CANONICAL_SLUG_RE .search (line )
247+ if match :
248+ return match .group (1 )
249+ except OSError :
250+ pass
251+ return None
252+
253+
254+ def write_sitemaps (output_dir ):
255+ """Write sitemap1.xml.. (both the /term/ page and the /reports/ viewer
256+ URL per term) and the sitemap-terms.xml index over them.
257+
258+ Call once, after every group has been processed, so the corpus is
259+ complete -- process_group() covers the full current ID list from Neo4j
260+ on every run, "skip" included, so by then _sitemap_entries holds every
261+ term regardless of whether this run touched its page.
262+ """
263+ if not output_dir :
264+ return
265+ with _sitemap_lock :
266+ entries = sorted (_sitemap_entries .items ())
267+ if not entries :
268+ print ("WARNING: no sitemap entries collected -- not writing sitemaps" )
269+ return
270+
271+ os .makedirs (output_dir , exist_ok = True )
272+ shard_names = []
273+ for shard_start in range (0 , len (entries ), SITEMAP_MAX_TERMS_PER_SHARD ):
274+ shard = entries [shard_start :shard_start + SITEMAP_MAX_TERMS_PER_SHARD ]
275+ shard_index = len (shard_names ) + 1
276+ shard_name = f"sitemap{ shard_index } .xml"
277+ lines = ['<?xml version="1.0" encoding="UTF-8"?>' ,
278+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' ]
279+ for term_id , url_slug in shard :
280+ lines .append (f' <url><loc>{ SITEMAP_SITE_BASE } /term/{ url_slug } /</loc></url>' )
281+ lines .append (f' <url><loc>{ SITEMAP_REPORT_BASE } /reports/{ term_id } </loc></url>' )
282+ lines .append ('</urlset>' )
283+ with open (join (output_dir , shard_name ), "w" , encoding = "utf-8" ) as f :
284+ f .write ("\n " .join (lines ) + "\n " )
285+ shard_names .append (shard_name )
286+
287+ index_lines = ['<?xml version="1.0" encoding="UTF-8"?>' ,
288+ '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' ]
289+ for shard_name in shard_names :
290+ index_lines .append (f' <sitemap><loc>{ SITEMAP_SITE_BASE } /{ shard_name } </loc></sitemap>' )
291+ index_lines .append ('</sitemapindex>' )
292+ with open (join (output_dir , "sitemap-terms.xml" ), "w" , encoding = "utf-8" ) as f :
293+ f .write ("\n " .join (index_lines ) + "\n " )
294+
295+ print (f"Wrote { len (shard_names )} sitemap shard(s) covering { len (entries )} terms to { output_dir } " )
296+
204297def get_query_results_url (term_id , query_name ):
205298 """Create a VFB viewer URL for a term query result set."""
206299 if not query_name :
@@ -860,13 +953,17 @@ def process_term(term_id):
860953 """
861954 filename = term_id + "_v" + str (version ) + ".md"
862955 if os .path .isfile (filename ):
956+ # Unchanged term: no re-fetch, so recover the slug already on disk
957+ # rather than skip the sitemap entry entirely.
958+ record_sitemap_entry (term_id , slug_from_existing_page (filename ))
863959 return "skip"
864960
865961 term_data = fetch_term_info (term_id )
866962 if term_data is None :
867963 return "fail"
868964
869965 page_content = generate_page (term_data )
966+ record_sitemap_entry (term_id , get_term_url (term_data .get ("Name" , "" ), term_id ))
870967
871968 tmp = f"{ filename } .{ os .getpid ()} .{ threading .get_ident ()} .tmp"
872969 with open (tmp , "w" , encoding = "utf-8" ) as f :
@@ -1145,6 +1242,13 @@ def test_medulla_page():
11451242if __name__ == "__main__" :
11461243 if len (sys .argv ) > 1 :
11471244 mypath = sys .argv [1 ]
1245+ # Optional: where to write sitemap1.xml.. and sitemap-terms.xml once
1246+ # the full corpus below has been processed. Point this at wherever
1247+ # is served as static files at the site root (e.g. the Hugo `public`
1248+ # directory the deploy already publishes from) -- robots.txt already
1249+ # advertises /sitemap-terms.xml, so dropping the files there is all
1250+ # that is needed for them to take effect.
1251+ sitemap_dir = sys .argv [2 ] if len (sys .argv ) > 2 else None
11481252 print ("Updating all files in " + mypath )
11491253 groups = [
11501254 ('fbbt/' , 'FBbt classes' , "MATCH (n:Class) WHERE n.short_form starts with 'FBbt' WITH n.short_form as id ORDER BY id ASC RETURN collect(distinct id) as ids" ),
@@ -1174,6 +1278,11 @@ def test_medulla_page():
11741278 for relative_dir , label , query in groups :
11751279 process_group (mypath , relative_dir , label , query )
11761280
1281+ if sitemap_dir :
1282+ write_sitemaps (sitemap_dir )
1283+ else :
1284+ print ("No sitemap output dir given (3rd argument) -- skipping sitemap write." )
1285+
11771286 else :
11781287 print ("Testing term page generation..." )
11791288 success = test_medulla_page ()
0 commit comments