Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions .github/workflows/update-citations.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ jobs:
timeout-minutes: 60

steps:
- name: Debug dump
uses: crazy-max/ghaction-dump-context@v2

- name: Checkout branch contents
if: github.event.action != 'closed'
uses: actions/checkout@v4
Expand Down Expand Up @@ -58,7 +55,7 @@ jobs:
- name: Build updated citations
if: github.event.action != 'closed'
run: python _cite/cite.py
timeout-minutes: 15
timeout-minutes: 60

- name: Commit cache
if: failure()
Expand Down
Binary file removed _cite/.cache/cache.db
Binary file not shown.
61 changes: 51 additions & 10 deletions _cite/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

import subprocess
import json
from urllib.parse import quote
from urllib.request import Request, urlopen
import yaml
from yaml.loader import SafeLoader
from pathlib import Path
Expand Down Expand Up @@ -171,6 +173,47 @@ def save_data(path, data):
except Exception:
raise Exception("Can't write to file")

def crossref_citation_fallback(_id):
"""Fallback to Crossref metadata when Manubot or DOI resolution fails."""
if not _id or not _id.lower().startswith("doi:"):
return None
doi = _id.split(":", 1)[1]
url = "https://api.crossref.org/works/" + quote(doi)
request = Request(
url=url,
headers={"User-Agent": "mmv-lab.github.io-citation-bot/1.0"},
)
try:
with urlopen(request, timeout=30) as response:
payload = json.load(response)
except Exception:
return None

message = get_safe(payload, "message", {})
title = get_safe(message, "title", [""])[0]
authors = get_safe(message, "author", [])
issued = get_safe(message, "published-print", get_safe(message, "published-online", get_safe(message, "issued", {})))
date_parts = get_safe(issued, "date-parts", [[None]])
year = date_parts[0][0] if date_parts and date_parts[0] else ""
month = date_parts[0][1] if len(date_parts[0]) > 1 else "1"
day = date_parts[0][2] if len(date_parts[0]) > 2 else "1"
Comment on lines +198 to +199

citation = {
"title": title or "",
"author": [
{
"given": get_safe(author, "given", "").strip(),
"family": get_safe(author, "family", "").strip(),
}
for author in authors
],
"container-title": get_safe(message, "container-title", [""])[0],
"publisher": get_safe(message, "publisher", ""),
"URL": get_safe(message, "URL", ""),
"issued": {"date-parts": [[year, month, day]] if year else [[None, None, None]]},
}
return citation


@log_cache
@cache.memoize(name="manubot", expire=90 * (60 * 60 * 24))
Expand All @@ -179,19 +222,17 @@ def cite_with_manubot(_id):
generate citation data for source id with Manubot
"""

# run Manubot
try:
commands = ["manubot", "cite", _id, "--log-level=WARNING"]
output = subprocess.Popen(commands, stdout=subprocess.PIPE).communicate()
except Exception as e:
log(e, indent=3)
raise Exception("Manubot could not generate citation")

# parse results as json
try:
manubot = json.loads(output[0])[0]
output = subprocess.Popen(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
stdout = output[0] or b""
if not stdout:
raise ValueError("No stdout from Manubot")
manubot = json.loads(stdout)[0]
except Exception:
raise Exception("Couldn't parse Manubot response")
manubot = crossref_citation_fallback(_id)
if not manubot:
raise Exception("Manubot could not generate citation")
Comment on lines +227 to +235

# new citation with only needed info
citation = {}
Expand Down
Loading