Skip to content

Commit e091df7

Browse files
Index Paper Citations (#14)
* docs: plan evidence-backed paper citation indexing * feat(db): add durable citation evidence and leased jobs * feat(citations): resolve identities and fetch provider references * feat(citations): add resumable indexing and import capture * feat(api): expose local citations and indexing status * feat(ui): show paper references and incoming citations * docs(citations): record coverage pilot and operating procedures * feat(db): version PDF replacements and add opt-in citation jobs * feat(citations): extract evidenced PDF references with GROBID * feat(citations): expose opt-in PDF indexing and pilot controls * feat(db): add independently opted-in local PDF citation jobs * feat(citations): extract numbered PDF bibliographies locally * docs(citations): record local PDF pilot and revise rollout plan * fix(ui): remove citation implementation notes * fix(citations): distinguish retrieved references from library links * fix(db): upgrade earlier citation identity installations safely * feat(db): guard automatic PDF citation fallback after provider attempts * feat(citations): automatically schedule local PDF fallback for imports * fix(ui): consolidate citation indexing controls * style(citations): align Index with header and lab button styling * style(citations): reduce note and paper title text size * style(citations): simplify reference list and space unmatched count * style(citations): match detail headings and remove divider
2 parents 4e5e53d + ff38802 commit e091df7

57 files changed

Lines changed: 3861 additions & 1 deletion

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/tests.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,18 @@ on:
66
branches: [main]
77

88
jobs:
9+
citation-database:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v4
13+
- uses: actions/setup-node@v4
14+
with:
15+
node-version: "20"
16+
- name: Install disposable PostgreSQL engine
17+
run: npm install --prefix backend/data/citation-validation --no-audit --no-fund --ignore-scripts @electric-sql/pglite@0.5.14
18+
- name: Verify citation migration and transactions
19+
run: node backend/tests/run_citation_sql.mjs
20+
921
backend:
1022
runs-on: ubuntu-latest
1123
defaults:

‎backend/.env.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
OPENAI_API_KEY=*
22
OPENALEX_API_KEY=*
3+
# Optional alternate extractor; local PyMuPDF indexing needs no service.
4+
# PDFs are sent here only for explicitly selected GROBID jobs.
5+
GROBID_URL=
36
SUPABASE_URL=*
47
SUPABASE_KEY=*
58

‎backend/app.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from routers import papers, collections, workflows, runs, proposals, activity, search, libraries, websites, notes, chat, authors, github_repos, settings, notes_copilot, usage, projects, research_questions, experiments, project_notes_copilot, tasks, gap_analysis, batch
2020
from services.db import get_client
2121
from routers import labs
22+
from routers import citations
2223

2324
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
2425
logger = logging.getLogger(__name__)
@@ -46,6 +47,7 @@ async def storage_migration_maintenance(request: Request, call_next):
4647

4748
app.include_router(libraries.router)
4849
app.include_router(papers.router)
50+
app.include_router(citations.router)
4951
app.include_router(websites.router)
5052
app.include_router(github_repos.router)
5153
app.include_router(collections.router)

‎backend/migrations/033_paper_citations.sql‎

Lines changed: 330 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
-- Optional PDF bibliography indexing. Requires migration 033.
2+
BEGIN;
3+
ALTER TABLE public.papers ADD COLUMN IF NOT EXISTS pdf_revision BIGINT NOT NULL DEFAULT 0;
4+
ALTER TABLE public.paper_reference_runs DROP CONSTRAINT IF EXISTS paper_reference_runs_provider_check;
5+
ALTER TABLE public.paper_reference_runs ADD CONSTRAINT paper_reference_runs_provider_check
6+
CHECK (provider IN ('openalex','crossref','grobid'));
7+
ALTER TABLE public.paper_citation_jobs DROP CONSTRAINT IF EXISTS paper_citation_jobs_provider_check;
8+
ALTER TABLE public.paper_citation_jobs ADD CONSTRAINT paper_citation_jobs_provider_check
9+
CHECK (provider IN ('openalex','crossref','grobid'));
10+
11+
CREATE OR REPLACE FUNCTION public.citation_fingerprint(p public.papers) RETURNS TEXT
12+
LANGUAGE sql IMMUTABLE AS $$
13+
SELECT md5(jsonb_build_array(p.doi,p.arxiv_id,p.title,p.authors,p.year,p.library_id,p.pdf_url,p.pdf_revision)::text);
14+
$$;
15+
16+
CREATE OR REPLACE FUNCTION public.bump_citation_pdf_revision() RETURNS trigger
17+
LANGUAGE plpgsql AS $$
18+
BEGIN
19+
-- The upload route writes pdf_url even when replacing the same storage key.
20+
NEW.pdf_revision := OLD.pdf_revision + 1;
21+
RETURN NEW;
22+
END $$;
23+
DROP TRIGGER IF EXISTS citation_pdf_revision ON public.papers;
24+
CREATE TRIGGER citation_pdf_revision BEFORE UPDATE OF pdf_url ON public.papers
25+
FOR EACH ROW EXECUTE FUNCTION public.bump_citation_pdf_revision();
26+
27+
CREATE OR REPLACE FUNCTION public.enqueue_pdf_citations(p_paper_id TEXT,p_force BOOLEAN DEFAULT false)
28+
RETURNS SETOF public.paper_citation_jobs
29+
LANGUAGE plpgsql SECURITY INVOKER SET search_path=public AS $$
30+
DECLARE p papers;
31+
BEGIN
32+
SELECT * INTO p FROM papers WHERE id=p_paper_id FOR UPDATE;
33+
IF NOT FOUND THEN RETURN; END IF;
34+
INSERT INTO paper_citation_jobs(paper_id,provider,fingerprint)
35+
VALUES (p.id,'grobid',citation_fingerprint(p))
36+
ON CONFLICT (paper_id,provider) DO UPDATE SET
37+
fingerprint=EXCLUDED.fingerprint,status='pending',attempts=0,error=NULL,
38+
lease_token=NULL,lease_until=NULL,next_attempt_at=now(),updated_at=now()
39+
WHERE paper_citation_jobs.fingerprint<>EXCLUDED.fingerprint
40+
OR (p_force AND paper_citation_jobs.status NOT IN ('pending','running'));
41+
RETURN QUERY SELECT * FROM paper_citation_jobs WHERE paper_id=p.id AND provider='grobid';
42+
END $$;
43+
44+
CREATE OR REPLACE FUNCTION public.refresh_opted_in_pdf_citations() RETURNS trigger
45+
LANGUAGE plpgsql SECURITY INVOKER SET search_path=public AS $$
46+
BEGIN
47+
IF citation_fingerprint(OLD)<>citation_fingerprint(NEW)
48+
AND EXISTS(SELECT 1 FROM paper_citation_jobs WHERE paper_id=NEW.id AND provider='grobid') THEN
49+
PERFORM enqueue_pdf_citations(NEW.id);
50+
END IF;
51+
RETURN NEW;
52+
END $$;
53+
DROP TRIGGER IF EXISTS paper_pdf_citations_changed ON public.papers;
54+
CREATE TRIGGER paper_pdf_citations_changed AFTER UPDATE ON public.papers
55+
FOR EACH ROW EXECUTE FUNCTION public.refresh_opted_in_pdf_citations();
56+
57+
-- Rebase existing queued fingerprints without disturbing published provider evidence.
58+
UPDATE paper_citation_jobs j SET fingerprint=citation_fingerprint(p),status='pending',attempts=0,
59+
lease_token=NULL,lease_until=NULL,next_attempt_at=now(),updated_at=now()
60+
FROM papers p WHERE p.id=j.paper_id AND j.fingerprint<>citation_fingerprint(p);
61+
COMMIT;
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
-- Local PDF extraction, opt-in independently of GROBID. Requires 033 and 034.
2+
BEGIN;
3+
ALTER TABLE public.paper_reference_runs DROP CONSTRAINT IF EXISTS paper_reference_runs_provider_check;
4+
ALTER TABLE public.paper_reference_runs ADD CONSTRAINT paper_reference_runs_provider_check CHECK (provider IN ('openalex','crossref','grobid','pymupdf'));
5+
ALTER TABLE public.paper_citation_jobs DROP CONSTRAINT IF EXISTS paper_citation_jobs_provider_check;
6+
ALTER TABLE public.paper_citation_jobs ADD CONSTRAINT paper_citation_jobs_provider_check CHECK (provider IN ('openalex','crossref','grobid','pymupdf'));
7+
CREATE OR REPLACE FUNCTION public.enqueue_local_pdf_citations(p_paper_id TEXT,p_force BOOLEAN DEFAULT false)
8+
RETURNS SETOF public.paper_citation_jobs
9+
LANGUAGE plpgsql SECURITY INVOKER SET search_path=public AS $$
10+
DECLARE p papers;
11+
BEGIN
12+
SELECT * INTO p FROM papers WHERE id=p_paper_id FOR UPDATE;
13+
IF NOT FOUND THEN RETURN; END IF;
14+
INSERT INTO paper_citation_jobs(paper_id,provider,fingerprint)
15+
VALUES (p.id,'pymupdf',citation_fingerprint(p))
16+
ON CONFLICT (paper_id,provider) DO UPDATE SET
17+
fingerprint=EXCLUDED.fingerprint,status='pending',attempts=0,error=NULL,
18+
lease_token=NULL,lease_until=NULL,next_attempt_at=now(),updated_at=now()
19+
WHERE paper_citation_jobs.fingerprint<>EXCLUDED.fingerprint
20+
OR (p_force AND paper_citation_jobs.status NOT IN ('pending','running'));
21+
RETURN QUERY SELECT * FROM paper_citation_jobs WHERE paper_id=p.id AND provider='pymupdf';
22+
END $$;
23+
24+
CREATE OR REPLACE FUNCTION public.refresh_opted_in_pdf_citations() RETURNS trigger
25+
LANGUAGE plpgsql SECURITY INVOKER SET search_path=public AS $$
26+
BEGIN
27+
IF citation_fingerprint(OLD)<>citation_fingerprint(NEW) THEN
28+
IF EXISTS(SELECT 1 FROM paper_citation_jobs WHERE paper_id=NEW.id AND provider='grobid') THEN
29+
PERFORM enqueue_pdf_citations(NEW.id);
30+
END IF;
31+
IF EXISTS(SELECT 1 FROM paper_citation_jobs WHERE paper_id=NEW.id AND provider='pymupdf') THEN
32+
PERFORM enqueue_local_pdf_citations(NEW.id);
33+
END IF;
34+
END IF;
35+
RETURN NEW;
36+
END $$;
37+
COMMIT;

0 commit comments

Comments
 (0)