55Shared utilities for fetch_orcid.py, fetch_pubmed.py, fetch_scholar.py.
66
77Provides:
8+ - is_manual(entry)
89 - load_manual_fingerprints(bib_path)
910 - fingerprint_matches(entry, fp)
11+ - display_entry(bibtex, index, total, source)
12+ - edit_in_editor(bibtex)
13+ - prompt_action(has_doi)
14+ - interactive_review(candidates, rejected_file)
1015
11- These functions live here (not in update_refs.py) so each fetch script
12- can import them when run standalone as well as when invoked as a subprocess
13- by update_refs.py.
16+ These live here so each fetch script can use them both when run standalone
17+ and when called in-process by update_refs.py.
1418"""
1519
20+ import json
21+ import os
1622import re
23+ import subprocess
24+ import sys
25+ import tempfile
26+ import webbrowser
1727from pathlib import Path
1828
1929
30+ # ---------------------------------------------------------------------------
31+ # MANUAL ENTRY PROTECTION
32+ # ---------------------------------------------------------------------------
33+
2034def is_manual (entry : dict ) -> bool :
2135 """Return True if a bibtexparser entry is flagged as manual."""
2236 return "manual" in entry .get ("keywords" , "" ).lower ()
@@ -26,7 +40,6 @@ def load_manual_fingerprints(bib_path: Path) -> dict:
2640 """
2741 Parse a .bib file and return fingerprints of all manual entries.
2842 Returns dict with sets: {keys, dois, pmids, titles}
29- Used by fetch scripts to avoid appending duplicates of manual entries.
3043 """
3144 fp = {"keys" : set (), "dois" : set (), "pmids" : set (), "titles" : set ()}
3245 if not bib_path .exists ():
@@ -47,11 +60,11 @@ def load_manual_fingerprints(bib_path: Path) -> dict:
4760 doi = entry .get ("doi" , "" ).lower ().strip ()
4861 if doi :
4962 fp ["dois" ].add (doi )
50- pmid_match = re .search (r' PMID:\s*(\d+)' ,
63+ pmid_match = re .search (r" PMID:\s*(\d+)" ,
5164 entry .get ("note" , "" ), re .IGNORECASE )
5265 if pmid_match :
5366 fp ["pmids" ].add (pmid_match .group (1 ))
54- title = re .sub (r' \s+' , ' ' , entry .get ("title" , "" )).lower ().strip ()
67+ title = re .sub (r" \s+" , " " , entry .get ("title" , "" )).lower ().strip ()
5568 if title :
5669 fp ["titles" ].add (title )
5770
@@ -62,25 +75,22 @@ def _load_manual_fingerprints_regex(bib_path: Path) -> dict:
6275 """Regex fallback for load_manual_fingerprints (no bibtexparser)."""
6376 fp = {"keys" : set (), "dois" : set (), "pmids" : set (), "titles" : set ()}
6477 text = bib_path .read_text (encoding = "utf-8" )
65- entries = re .split (r' \n(?=@)' , text )
78+ entries = re .split (r" \n(?=@)" , text )
6679
6780 for entry in entries :
68- kw = re .search (r' keywords\s*=\s*\{([^}]*)\}' , entry , re .IGNORECASE )
81+ kw = re .search (r" keywords\s*=\s*\{([^}]*)\}" , entry , re .IGNORECASE )
6982 if not kw or "manual" not in kw .group (1 ).lower ():
7083 continue
71- key_m = re .search (r'@\w+\{(\S+),' , entry )
72- doi_m = re .search (r'doi\s*=\s*\{([^}]+)\}' , entry , re .IGNORECASE )
73- pmid_m = re .search (r'PMID:\s*(\d+)' , entry , re .IGNORECASE )
74- title_m = re .search (r'title\s*=\s*\{([^}]+)\}' , entry , re .IGNORECASE )
75-
76- if key_m :
77- fp ["keys" ].add (key_m .group (1 ).lower ())
78- if doi_m :
79- fp ["dois" ].add (doi_m .group (1 ).lower ().strip ())
80- if pmid_m :
81- fp ["pmids" ].add (pmid_m .group (1 ))
84+ key_m = re .search (r"@\w+\{(\S+)," , entry )
85+ doi_m = re .search (r"doi\s*=\s*\{([^}]+)\}" , entry , re .IGNORECASE )
86+ pmid_m = re .search (r"PMID:\s*(\d+)" , entry , re .IGNORECASE )
87+ title_m = re .search (r"title\s*=\s*\{([^}]+)\}" , entry , re .IGNORECASE )
88+
89+ if key_m : fp ["keys" ].add (key_m .group (1 ).lower ())
90+ if doi_m : fp ["dois" ].add (doi_m .group (1 ).lower ().strip ())
91+ if pmid_m : fp ["pmids" ].add (pmid_m .group (1 ))
8292 if title_m :
83- title = re .sub (r' \s+' , ' ' , title_m .group (1 )).lower ().strip ()
93+ title = re .sub (r" \s+" , " " , title_m .group (1 )).lower ().strip ()
8494 fp ["titles" ].add (title )
8595
8696 return fp
@@ -96,12 +106,278 @@ def fingerprint_matches(entry: dict, fp: dict) -> bool:
96106 doi = entry .get ("doi" , "" ).lower ().strip ()
97107 if doi and doi in fp ["dois" ]:
98108 return True
99- pmid_match = re .search (r' PMID:\s*(\d+)' ,
109+ pmid_match = re .search (r" PMID:\s*(\d+)" ,
100110 entry .get ("note" , "" ), re .IGNORECASE )
101111 if pmid_match and pmid_match .group (1 ) in fp ["pmids" ]:
102112 return True
103- title = re .sub (r'\s+' , ' ' ,
104- entry .get ("title" , "" )).lower ().strip ()
113+ title = re .sub (r"\s+" , " " , entry .get ("title" , "" )).lower ().strip ()
105114 if title and title in fp ["titles" ]:
106115 return True
107- return False
116+ return False
117+
118+
119+ # ---------------------------------------------------------------------------
120+ # INTERACTIVE REVIEW
121+ # ---------------------------------------------------------------------------
122+
123+ def load_rejected (path : Path ) -> dict :
124+ """Load persistent rejection list {key: title} from a JSON file."""
125+ if not path .exists ():
126+ return {}
127+ try :
128+ return json .loads (path .read_text (encoding = "utf-8" ))
129+ except Exception :
130+ return {}
131+
132+
133+ def save_rejected (path : Path , rejected : dict ) -> None :
134+ """Save rejection list to JSON, creating parent dirs if needed."""
135+ path .parent .mkdir (parents = True , exist_ok = True )
136+ path .write_text (
137+ json .dumps (rejected , indent = 2 , ensure_ascii = False ),
138+ encoding = "utf-8" ,
139+ )
140+
141+
142+ def display_entry (bibtex : str , index : int , total : int , source : str ) -> None :
143+ """Print a BibTeX entry with a header line for review."""
144+ W = 72
145+ print ()
146+ print ("-" * W )
147+ print (f" Entry { index } of { total } [{ source } ]" )
148+ print ("-" * W )
149+ for line in bibtex .splitlines ():
150+ print (" " + line )
151+ print ()
152+
153+
154+ def edit_in_editor (bibtex : str ) -> str :
155+ """Open bibtex in $EDITOR (fallback: nano) and return the edited result."""
156+ editor = os .environ .get ("EDITOR" ) or os .environ .get ("VISUAL" ) or "nano"
157+ with tempfile .NamedTemporaryFile (
158+ mode = "w" , suffix = ".bib" , delete = False , encoding = "utf-8"
159+ ) as f :
160+ f .write (bibtex )
161+ tmp = Path (f .name )
162+ try :
163+ subprocess .run ([editor , str (tmp )], check = True )
164+ return tmp .read_text (encoding = "utf-8" ).strip ()
165+ except Exception as exc :
166+ print (f" [warn] Editor failed ({ exc } ); keeping original." )
167+ return bibtex
168+ finally :
169+ tmp .unlink (missing_ok = True )
170+
171+
172+ def prompt_action (has_doi : bool , allow_unreject : bool = False ) -> str :
173+ """
174+ Prompt for a review action. Returns one of:
175+ accept edit keyword manual reject unreject skip open quit
176+ unreject is only offered when allow_unreject=True (--review-rejected mode).
177+ """
178+ line = " [a]ccept [e]dit [k]eywords [m]anual [r]eject+remember [s]kip"
179+ if allow_unreject :
180+ line += " [u]nreject"
181+ if has_doi :
182+ line += " [o]pen DOI"
183+ line += " : "
184+
185+ while True :
186+ try :
187+ ch = input (line ).strip ().lower ()
188+ except (EOFError , KeyboardInterrupt ):
189+ print ("\n Interrupted." )
190+ return "quit"
191+
192+ if ch in ("a" , "accept" ): return "accept"
193+ if ch in ("e" , "edit" ): return "edit"
194+ if ch in ("k" , "keywords" , "keyword" ): return "keyword"
195+ if ch in ("m" , "manual" ): return "manual"
196+ if ch in ("r" , "reject" ): return "reject"
197+ if ch in ("u" , "unreject" ) and allow_unreject : return "unreject"
198+ if ch in ("s" , "skip" ): return "skip"
199+ if ch in ("o" , "open" ) and has_doi : return "open"
200+ valid = "a/e/k/m/r/s" + ("/u" if allow_unreject else "" ) + ("/o" if has_doi else "" )
201+ print (f" Please enter one of: { valid } " )
202+
203+
204+
205+ def prompt_keywords (current : str = "" ) -> str :
206+ """
207+ Prompt for comma-separated keyword tags interactively.
208+ "manual" is always preserved when present. Returns the full keywords string.
209+ Leave blank to keep current tags. Enter "-" to clear all non-manual tags.
210+ """
211+ existing = [k .strip () for k in current .split ("," ) if k .strip ()]
212+ existing_display = ", " .join (existing ) if existing else "(none)"
213+ print (f" Current keywords: { existing_display } " )
214+ print (" Add tags (comma-separated). Leave blank to keep current. Enter - to clear." )
215+ try :
216+ raw = input (" Tags: " ).strip ()
217+ except (EOFError , KeyboardInterrupt ):
218+ return current
219+
220+ has_manual = "manual" in {k .lower () for k in existing }
221+
222+ if raw == "-" :
223+ return "manual" if has_manual else ""
224+ if not raw :
225+ return current
226+
227+ new_tags = [t .strip () for t in raw .split ("," ) if t .strip ()]
228+ all_tags = set (existing ) | set (new_tags )
229+ return ", " .join (sorted (all_tags ))
230+
231+
232+ def inject_keywords (bibtex : str , keywords : str ) -> str :
233+ """Replace the keywords field value in a BibTeX entry string."""
234+ if re .search (r"keywords\s*=\s*\{[^}]*\}" , bibtex , re .IGNORECASE ):
235+ return re .sub (
236+ r"keywords\s*=\s*\{[^}]*\}" ,
237+ f"keywords = {{{ keywords } }}" ,
238+ bibtex ,
239+ flags = re .IGNORECASE ,
240+ )
241+ # No keywords field present — insert before closing brace
242+ return bibtex .rstrip ().rstrip ("}" ) + f" keywords = {{{ keywords } }},\n }}"
243+
244+
245+
246+ def review_rejected (rejected_file : Path ) -> tuple :
247+ """
248+ Load the rejection list for a source and interactively review each entry.
249+ Entries can be un-rejected (removed from the list) or kept.
250+
251+ Returns (unrejected_keys, updated_rejected_dict).
252+ Callers should save the updated dict and optionally re-fetch un-rejected entries.
253+ """
254+ rejected = load_rejected (rejected_file )
255+ if not rejected :
256+ print (f" No rejected entries in { rejected_file } ." )
257+ return [], {}
258+
259+ print (f" Reviewing { len (rejected )} rejected entry/entries from { rejected_file } " )
260+ print (" For each entry: [k]eep rejected [u]nreject (remove from list) [q]uit" )
261+
262+ unrejected = []
263+ updated = dict (rejected )
264+ items = list (rejected .items ())
265+ total = len (items )
266+
267+ for i , (key , title ) in enumerate (items , 1 ):
268+ print ()
269+ print (f" [{ i } /{ total } ] key: { key } " )
270+ print (f" title: { title [:100 ]} " )
271+ while True :
272+ try :
273+ ch = input (" [k]eep [u]nreject [q]uit : " ).strip ().lower ()
274+ except (EOFError , KeyboardInterrupt ):
275+ print ("\n Interrupted." )
276+ return unrejected , updated
277+ if ch in ("k" , "keep" , "" ):
278+ print (" -> Kept rejected." )
279+ break
280+ if ch in ("u" , "unreject" ):
281+ del updated [key ]
282+ unrejected .append (key )
283+ print (" -> Un-rejected." )
284+ break
285+ if ch in ("q" , "quit" ):
286+ print (f" Stopped. { len (unrejected )} un-rejected so far." )
287+ return unrejected , updated
288+ print (" Please enter k, u, or q." )
289+
290+ print ()
291+ return unrejected , updated
292+
293+ def interactive_review (
294+ candidates : list , # list of (bibtex_str, doi_or_empty, reject_key, source_label)
295+ rejected_file : Path , # path to .json rejection list for this source
296+ allow_unreject : bool = False , # True when called from --review-rejected
297+ ) -> tuple :
298+ """
299+ Walk through candidate BibTeX entries interactively.
300+
301+ Each candidate is a tuple:
302+ (bibtex_str, doi, reject_key, source_label)
303+
304+ where reject_key is any stable string identifying the entry for the
305+ rejection list (e.g. PMID, DOI, or cite key).
306+
307+ Returns:
308+ (list_of_accepted_bibtex_strings, new_rejections_dict)
309+
310+ Rejected entries are not written to disk here — the caller merges
311+ new_rejections into the existing list and saves it.
312+ """
313+ rejected = load_rejected (rejected_file )
314+ accepted = []
315+ to_reject = {}
316+ total = len (candidates )
317+
318+ for i , (bibtex , doi , reject_key , source ) in enumerate (candidates , 1 ):
319+ # Skip anything already in the rejection list
320+ if reject_key in rejected :
321+ print (f" [skip] Previously rejected: { reject_key } " )
322+ continue
323+
324+ has_doi = bool (doi )
325+
326+ # Inner loop: open/edit re-display without advancing
327+ while True :
328+ display_entry (bibtex , i , total , source )
329+ action = prompt_action (has_doi , allow_unreject = allow_unreject )
330+
331+ if action == "open" :
332+ url = f"https://doi.org/{ doi } "
333+ print (f" Opening { url } ..." )
334+ webbrowser .open (url )
335+ continue
336+
337+ if action == "edit" :
338+ bibtex = edit_in_editor (bibtex )
339+ continue
340+
341+ if action == "keyword" :
342+ kw_m = re .search (r"keywords\s*=\s*\{([^}]*)\}" , bibtex , re .IGNORECASE )
343+ current = kw_m .group (1 ) if kw_m else ""
344+ new_kw = prompt_keywords (current )
345+ bibtex = inject_keywords (bibtex , new_kw )
346+ continue
347+
348+ break
349+
350+ if action == "accept" :
351+ accepted .append (bibtex ) # plain str
352+ print (" -> Accepted." )
353+
354+ elif action == "manual" :
355+ # Ensure "manual" is in keywords; preserve any tags already set via [k]
356+ kw_m = re .search (r"keywords\s*=\s*\{([^}]*)\}" , bibtex , re .IGNORECASE )
357+ current = kw_m .group (1 ) if kw_m else ""
358+ tags = {"manual" } | {t .strip () for t in current .split ("," ) if t .strip ()}
359+ bibtex = inject_keywords (bibtex , ", " .join (sorted (tags )))
360+ accepted .append (bibtex )
361+ print (" -> Accepted as manual." )
362+
363+ elif action == "reject" :
364+ # Extract a short title for the rejection record if possible
365+ title_m = re .search (r"title\s*=\s*\{([^}]+)\}" , bibtex , re .IGNORECASE )
366+ label = title_m .group (1 )[:120 ] if title_m else reject_key
367+ to_reject [reject_key ] = label
368+ print (" -> Rejected and remembered." )
369+
370+ elif action == "skip" :
371+ print (" -> Skipped (will appear again next run)." )
372+
373+ elif action == "unreject" :
374+ # Will be removed from rejection list by caller
375+ accepted .append (("__unreject__" , reject_key ))
376+ print (" -> Un-rejected (removed from rejection list)." )
377+
378+ elif action == "quit" :
379+ print (f" Stopped at entry { i } . { len (accepted )} accepted so far." )
380+ break
381+
382+ print ()
383+ return accepted , to_reject
0 commit comments