Skip to content

Commit 570ae7e

Browse files
feat(migration): inventory and verify paper PDF transfers
Inventory paginated paper records and storage objects, then copy referenced PDFs with dry-run selection, source-change checks, no-overwrite uploads, checksum verification, and resumable journals.
1 parent f7702e1 commit 570ae7e

4 files changed

Lines changed: 462 additions & 0 deletions

File tree

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
"""Copy referenced PDFs to GCS and verify bytes. Never updates URLs or deletes files."""
2+
from __future__ import annotations
3+
4+
import argparse
5+
from concurrent.futures import ThreadPoolExecutor, as_completed
6+
from datetime import datetime, timezone
7+
import hashlib
8+
import json
9+
import os
10+
from pathlib import Path
11+
import tempfile
12+
13+
from inventory_paper_storage import source_key
14+
15+
16+
def lookup_source(client, bucket, key):
17+
"""Read one folder with a filename search instead of the whole bucket."""
18+
parent, _, name = key.rpartition("/")
19+
offset = 0
20+
while True:
21+
page = client.storage.from_(bucket).list(parent, {
22+
"search": name, "limit": 100, "offset": offset,
23+
"sortBy": {"column": "name", "order": "asc"},
24+
})
25+
if not page:
26+
return None
27+
for obj in page:
28+
if obj["name"] == name:
29+
return {**obj, "key": key}
30+
offset += len(page)
31+
32+
33+
def fingerprint(obj):
34+
metadata = obj.get("metadata") or {}
35+
return (obj.get("id"), obj.get("version"), obj.get("updated_at"),
36+
metadata.get("size"), metadata.get("eTag"))
37+
38+
39+
def digest(path):
40+
result = hashlib.sha256()
41+
with path.open("rb") as handle:
42+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
43+
result.update(chunk)
44+
return result.hexdigest()
45+
46+
47+
def select_entries(report, limit):
48+
objects = {obj["key"]: obj for obj in report["objects"]}
49+
entries = []
50+
seen = set()
51+
for entry in report["managed_references"]:
52+
key = entry["source_key"]
53+
if key in objects and key not in seen and key.lower().endswith(".pdf"):
54+
seen.add(key)
55+
entries.append({**entry, "object": objects[key]})
56+
entries.sort(key=lambda entry: (int((entry["object"].get("metadata") or {}).get("size") or 0), entry["source_key"]))
57+
if limit == 3 and len(entries) > 3:
58+
return [entries[0], entries[len(entries) // 2], entries[-1]]
59+
return entries if limit == 0 else entries[:limit]
60+
61+
62+
def copy_one(entry, bucket, http, source_client, source_bucket, base_url, previous=None):
63+
key = entry["source_key"]
64+
if source_key(entry["pdf_url"], base_url, source_bucket) != key:
65+
raise ValueError("Source URL does not match the configured project/bucket/key")
66+
# Check the current database reference before copying an inventoried file.
67+
rows = source_client.table("papers").select("id,pdf_url").eq("id", entry["id"]).execute().data
68+
if not rows or rows[0]["pdf_url"] != entry["pdf_url"]:
69+
raise ValueError("Paper was removed or its PDF reference changed; refresh inventory")
70+
before = lookup_source(source_client, source_bucket, key)
71+
if before is None or fingerprint(before) != fingerprint(entry["object"]):
72+
raise ValueError("Source changed since inventory; refresh inventory")
73+
if (previous and previous.get('status') == 'copied_verified'
74+
and previous.get('original_url') == entry['pdf_url']
75+
and previous.get('destination_bucket') == bucket.name
76+
and previous.get('destination_key') == key
77+
and tuple(previous['source_fingerprint']) == fingerprint(before)):
78+
existing = bucket.get_blob(key)
79+
if (existing is not None and int(existing.generation) == previous['destination_generation']
80+
and int(existing.size) == previous['bytes']
81+
and (existing.metadata or {}).get('researchos-source-sha256') == previous['sha256']):
82+
return {**previous, 'created': False, 'verification': 'previously-hashed-generation-unchanged'}
83+
with tempfile.TemporaryDirectory(prefix="researchos-pdf-") as tmp:
84+
source = Path(tmp) / "source.pdf"
85+
destination = Path(tmp) / "destination.pdf"
86+
expected_size = int(entry["object"]["metadata"]["size"])
87+
downloaded = 0
88+
with http.stream("GET", entry["pdf_url"]) as response:
89+
response.raise_for_status()
90+
with source.open("wb") as handle:
91+
for chunk in response.iter_bytes(1024 * 1024):
92+
downloaded += len(chunk)
93+
if downloaded > expected_size:
94+
raise ValueError("Source exceeds inventoried size")
95+
handle.write(chunk)
96+
with source.open("rb") as handle:
97+
if handle.read(5) != b"%PDF-":
98+
raise ValueError("Source does not have a PDF header")
99+
if downloaded != expected_size:
100+
raise ValueError("Source size differs from inventory")
101+
source_sha = digest(source)
102+
after = lookup_source(source_client, source_bucket, key)
103+
if after is None or fingerprint(after) != fingerprint(before):
104+
raise ValueError("Source changed during download")
105+
blob = bucket.get_blob(key)
106+
created = blob is None
107+
if created:
108+
blob = bucket.blob(key, chunk_size=8 * 1024 * 1024)
109+
blob.cache_control = "no-cache"
110+
blob.content_disposition = "inline"
111+
blob.metadata = {"researchos-source-sha256": source_sha}
112+
blob.upload_from_filename(str(source), content_type="application/pdf",
113+
if_generation_match=0, checksum="auto", timeout=120)
114+
generation = int(blob.generation)
115+
# Pin the read to the exact generation; never verify a different version.
116+
bucket.blob(key, generation=generation).download_to_filename(
117+
str(destination), if_generation_match=generation, checksum="auto", timeout=120)
118+
if destination.stat().st_size != downloaded or digest(destination) != source_sha:
119+
raise ValueError("Destination checksum/size mismatch; existing object was not overwritten")
120+
return {"paper_id": entry["id"], "source_key": key,
121+
"original_url": entry["pdf_url"], "source_fingerprint": fingerprint(after),
122+
"destination_bucket": bucket.name, "destination_key": key,
123+
"destination_generation": generation, "bytes": downloaded,
124+
"sha256": source_sha, "created": created, "status": "copied_verified"}
125+
126+
127+
def main():
128+
root = Path(__file__).resolve().parents[1]
129+
parser = argparse.ArgumentParser(description=__doc__)
130+
parser.add_argument("--inventory", type=Path, default=root / "data/migration/paper-inventory.json")
131+
parser.add_argument("--env-file", type=Path, default=root / ".env")
132+
parser.add_argument("--project", default="researchos-513104")
133+
parser.add_argument("--bucket", default="researchos-papers")
134+
parser.add_argument("--limit", type=int, default=3, help="Default: small/median/large pilot; 0: all referenced PDFs")
135+
parser.add_argument("--journal", type=Path, default=root / "data/migration/copy-journal.jsonl")
136+
parser.add_argument("--execute", action="store_true", help="Actually copy; otherwise show selection only")
137+
parser.add_argument("--workers", type=int, default=1, help="Concurrent transfers (1-8)")
138+
args = parser.parse_args()
139+
if args.limit < 0:
140+
parser.error("--limit must be nonnegative")
141+
if not 1 <= args.workers <= 8:
142+
parser.error('--workers must be between 1 and 8')
143+
report = json.loads(args.inventory.read_text(encoding="utf-8-sig"))
144+
entries = select_entries(report, args.limit)
145+
print(json.dumps({"mode": "copy-and-verify" if args.execute else "dry-run",
146+
"bucket": args.bucket, "selected": len(entries),
147+
"bytes": sum(int(e["object"]["metadata"]["size"]) for e in entries)}, indent=2), flush=True)
148+
if not args.execute:
149+
return 0
150+
from dotenv import load_dotenv
151+
from google.cloud import storage
152+
import httpx
153+
from supabase import create_client
154+
155+
load_dotenv(args.env_file, override=False)
156+
base_url = os.environ.get("SUPABASE_URL", "").rstrip("/")
157+
secret = os.environ.get("SUPABASE_KEY", "")
158+
if not base_url.startswith("https://") or not secret:
159+
parser.error("Configure SUPABASE_URL and SUPABASE_KEY locally")
160+
source = create_client(base_url, secret)
161+
target = storage.Client(project=args.project).bucket(args.bucket)
162+
args.journal.parent.mkdir(parents=True, exist_ok=True)
163+
previous = {}
164+
if args.journal.exists():
165+
for line in args.journal.read_text(encoding='utf-8').splitlines():
166+
record = json.loads(line)
167+
previous[record['source_key']] = record
168+
failed = False
169+
with httpx.Client(timeout=120, follow_redirects=False, headers={"Accept-Encoding": "identity"}) as http, ThreadPoolExecutor(max_workers=args.workers) as pool:
170+
futures = {pool.submit(copy_one, entry, target, http, source, report['source_bucket'], base_url,
171+
previous.get(entry['source_key'])): entry for entry in entries}
172+
for future in as_completed(futures):
173+
entry = futures[future]
174+
if future.cancelled():
175+
continue
176+
try:
177+
record = future.result()
178+
except Exception as error:
179+
# Avoid logging exception messages that can contain credential-bearing URLs.
180+
record = {"source_key": entry["source_key"], "status": "failed",
181+
"error_type": type(error).__name__}
182+
if isinstance(error, ValueError):
183+
record["detail"] = str(error)
184+
record["recorded_at"] = datetime.now(timezone.utc).isoformat()
185+
with args.journal.open("a", encoding="utf-8") as handle:
186+
handle.write(json.dumps(record) + "\n")
187+
handle.flush()
188+
os.fsync(handle.fileno())
189+
print(json.dumps({key: value for key, value in record.items()
190+
if key in ("source_key", "status", "bytes", "created", "error_type", "detail")}), flush=True)
191+
if record["status"] == "failed":
192+
failed = True
193+
for pending in futures:
194+
pending.cancel()
195+
return 1 if failed else 0
196+
197+
198+
if __name__ == "__main__":
199+
raise SystemExit(main())
Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
"""Read-only PDF inventory. Run with uv; see the migration plan for instructions."""
2+
3+
from __future__ import annotations
4+
5+
import argparse
6+
from collections import Counter
7+
from datetime import datetime, timezone
8+
import json
9+
import os
10+
from pathlib import Path
11+
from urllib.parse import unquote, urlsplit
12+
13+
14+
def source_key(url: str | None, base_url: str, bucket: str) -> str | None:
15+
if not url:
16+
return None
17+
parsed = urlsplit(url)
18+
base = urlsplit(base_url)
19+
prefix = f"/storage/v1/object/public/{bucket}/"
20+
if (parsed.scheme, parsed.netloc) != (base.scheme, base.netloc):
21+
return None
22+
if not parsed.path.startswith(prefix):
23+
return None
24+
return unquote(parsed.path[len(prefix):]) or None
25+
26+
27+
def list_papers(client, page_size: int = 500) -> list[dict]:
28+
rows = []
29+
while True:
30+
page = (
31+
client.table("papers")
32+
.select("id,pdf_url,library_id")
33+
.order("id")
34+
.range(len(rows), len(rows) + page_size - 1)
35+
.execute().data
36+
)
37+
if not page:
38+
return rows
39+
rows.extend(page)
40+
41+
42+
def list_objects(client, bucket: str, page_size: int = 100) -> list[dict]:
43+
objects = []
44+
prefixes = [""]
45+
visited = set()
46+
while prefixes:
47+
prefix = prefixes.pop()
48+
if prefix in visited:
49+
continue
50+
visited.add(prefix)
51+
offset = 0
52+
while True:
53+
page = client.storage.from_(bucket).list(
54+
prefix,
55+
{"limit": page_size, "offset": offset,
56+
"sortBy": {"column": "name", "order": "asc"}},
57+
)
58+
if not page:
59+
break
60+
for entry in page:
61+
key = f"{prefix}/{entry['name']}" if prefix else entry["name"]
62+
if entry.get("id") is None and entry.get("metadata") is None:
63+
prefixes.append(key)
64+
else:
65+
objects.append({**entry, "key": key})
66+
offset += len(page)
67+
return objects
68+
69+
70+
def build_report(papers: list[dict], objects: list[dict], base_url: str, bucket: str) -> dict:
71+
by_key = {obj["key"]: obj for obj in objects}
72+
references = Counter()
73+
managed = []
74+
missing = []
75+
external = 0
76+
for paper in papers:
77+
key = source_key(paper.get("pdf_url"), base_url, bucket)
78+
if key is None:
79+
external += bool(paper.get("pdf_url"))
80+
continue
81+
references[key] += 1
82+
entry = {**paper, "source_key": key, "source_exists": key in by_key}
83+
managed.append(entry)
84+
if key not in by_key:
85+
missing.append(entry)
86+
pdfs = [obj for obj in objects if obj["key"].lower().endswith(".pdf")]
87+
sizes = [int((obj.get("metadata") or {}).get("size") or 0) for obj in pdfs]
88+
return {
89+
"created_at": datetime.now(timezone.utc).isoformat(),
90+
"mode": "read-only-inventory",
91+
"source_bucket": bucket,
92+
"summary": {
93+
"paper_rows_visible_to_credentials": len(papers),
94+
"objects_visible_to_credentials": len(objects),
95+
"pdf_objects": len(pdfs),
96+
"pdf_bytes_from_metadata": sum(sizes),
97+
"largest_pdf_bytes": max(sizes, default=0),
98+
"managed_paper_references": len(managed),
99+
"external_or_unrecognized_pdf_urls": external,
100+
"missing_source_references": len(missing),
101+
"unreferenced_objects": sum(obj["key"] not in references for obj in objects),
102+
},
103+
"warnings": [
104+
"Inventory includes only rows/objects visible to the supplied credentials; RLS can hide data.",
105+
"No PDF bytes were downloaded or checksums verified.",
106+
"Live changes can affect paginated inventory; reconcile under a write pause before cutover.",
107+
"External/unrecognized URLs need review for custom domains or non-public source URLs.",
108+
],
109+
"papers": papers,
110+
"managed_references": managed,
111+
"missing_sources": missing,
112+
"shared_source_keys": {key: count for key, count in references.items() if count > 1},
113+
"unreferenced_objects": [obj for obj in objects if obj["key"] not in references],
114+
"objects": objects,
115+
}
116+
117+
118+
def main() -> int:
119+
parser = argparse.ArgumentParser(description=__doc__)
120+
parser.add_argument("--env-file", type=Path, default=Path(__file__).resolve().parents[1] / ".env")
121+
parser.add_argument("--bucket", default="pdfs")
122+
parser.add_argument("--output", type=Path, default=Path(__file__).resolve().parents[1] / "data" / "migration" / "paper-inventory.json")
123+
args = parser.parse_args()
124+
from dotenv import load_dotenv
125+
from supabase import create_client
126+
127+
load_dotenv(args.env_file, override=False)
128+
base_url = os.environ.get("SUPABASE_URL", "").rstrip("/")
129+
key = os.environ.get("SUPABASE_KEY", "")
130+
if not base_url or not key or base_url == "*" or key == "*":
131+
parser.error("Set SUPABASE_URL and SUPABASE_KEY in the environment or backend/.env. Never put credentials in command arguments.")
132+
if urlsplit(base_url).scheme != "https":
133+
parser.error("The cloud inventory requires an HTTPS SUPABASE_URL.")
134+
if args.output.exists():
135+
parser.error("Output already exists; choose a new --output to preserve the earlier inventory.")
136+
client = create_client(base_url, key)
137+
report = build_report(list_papers(client), list_objects(client, args.bucket), base_url, args.bucket)
138+
args.output.parent.mkdir(parents=True, exist_ok=True)
139+
with args.output.open("x", encoding="utf-8") as handle:
140+
json.dump(report, handle, indent=2, ensure_ascii=False)
141+
print(json.dumps(report["summary"], indent=2))
142+
print(f"Inventory saved to {args.output}")
143+
return 0
144+
145+
146+
if __name__ == "__main__":
147+
raise SystemExit(main())

0 commit comments

Comments
 (0)