|
| 1 | +"""Copy referenced PDFs to GCS and verify bytes. Never updates URLs or deletes files.""" |
| 2 | +from __future__ import annotations |
| 3 | + |
| 4 | +import argparse |
| 5 | +from concurrent.futures import ThreadPoolExecutor, as_completed |
| 6 | +from datetime import datetime, timezone |
| 7 | +import hashlib |
| 8 | +import json |
| 9 | +import os |
| 10 | +from pathlib import Path |
| 11 | +import tempfile |
| 12 | + |
| 13 | +from inventory_paper_storage import source_key |
| 14 | + |
| 15 | + |
| 16 | +def lookup_source(client, bucket, key): |
| 17 | + """Read one folder with a filename search instead of the whole bucket.""" |
| 18 | + parent, _, name = key.rpartition("/") |
| 19 | + offset = 0 |
| 20 | + while True: |
| 21 | + page = client.storage.from_(bucket).list(parent, { |
| 22 | + "search": name, "limit": 100, "offset": offset, |
| 23 | + "sortBy": {"column": "name", "order": "asc"}, |
| 24 | + }) |
| 25 | + if not page: |
| 26 | + return None |
| 27 | + for obj in page: |
| 28 | + if obj["name"] == name: |
| 29 | + return {**obj, "key": key} |
| 30 | + offset += len(page) |
| 31 | + |
| 32 | + |
| 33 | +def fingerprint(obj): |
| 34 | + metadata = obj.get("metadata") or {} |
| 35 | + return (obj.get("id"), obj.get("version"), obj.get("updated_at"), |
| 36 | + metadata.get("size"), metadata.get("eTag")) |
| 37 | + |
| 38 | + |
| 39 | +def digest(path): |
| 40 | + result = hashlib.sha256() |
| 41 | + with path.open("rb") as handle: |
| 42 | + for chunk in iter(lambda: handle.read(1024 * 1024), b""): |
| 43 | + result.update(chunk) |
| 44 | + return result.hexdigest() |
| 45 | + |
| 46 | + |
| 47 | +def select_entries(report, limit): |
| 48 | + objects = {obj["key"]: obj for obj in report["objects"]} |
| 49 | + entries = [] |
| 50 | + seen = set() |
| 51 | + for entry in report["managed_references"]: |
| 52 | + key = entry["source_key"] |
| 53 | + if key in objects and key not in seen and key.lower().endswith(".pdf"): |
| 54 | + seen.add(key) |
| 55 | + entries.append({**entry, "object": objects[key]}) |
| 56 | + entries.sort(key=lambda entry: (int((entry["object"].get("metadata") or {}).get("size") or 0), entry["source_key"])) |
| 57 | + if limit == 3 and len(entries) > 3: |
| 58 | + return [entries[0], entries[len(entries) // 2], entries[-1]] |
| 59 | + return entries if limit == 0 else entries[:limit] |
| 60 | + |
| 61 | + |
| 62 | +def copy_one(entry, bucket, http, source_client, source_bucket, base_url, previous=None): |
| 63 | + key = entry["source_key"] |
| 64 | + if source_key(entry["pdf_url"], base_url, source_bucket) != key: |
| 65 | + raise ValueError("Source URL does not match the configured project/bucket/key") |
| 66 | + # Check the current database reference before copying an inventoried file. |
| 67 | + rows = source_client.table("papers").select("id,pdf_url").eq("id", entry["id"]).execute().data |
| 68 | + if not rows or rows[0]["pdf_url"] != entry["pdf_url"]: |
| 69 | + raise ValueError("Paper was removed or its PDF reference changed; refresh inventory") |
| 70 | + before = lookup_source(source_client, source_bucket, key) |
| 71 | + if before is None or fingerprint(before) != fingerprint(entry["object"]): |
| 72 | + raise ValueError("Source changed since inventory; refresh inventory") |
| 73 | + if (previous and previous.get('status') == 'copied_verified' |
| 74 | + and previous.get('original_url') == entry['pdf_url'] |
| 75 | + and previous.get('destination_bucket') == bucket.name |
| 76 | + and previous.get('destination_key') == key |
| 77 | + and tuple(previous['source_fingerprint']) == fingerprint(before)): |
| 78 | + existing = bucket.get_blob(key) |
| 79 | + if (existing is not None and int(existing.generation) == previous['destination_generation'] |
| 80 | + and int(existing.size) == previous['bytes'] |
| 81 | + and (existing.metadata or {}).get('researchos-source-sha256') == previous['sha256']): |
| 82 | + return {**previous, 'created': False, 'verification': 'previously-hashed-generation-unchanged'} |
| 83 | + with tempfile.TemporaryDirectory(prefix="researchos-pdf-") as tmp: |
| 84 | + source = Path(tmp) / "source.pdf" |
| 85 | + destination = Path(tmp) / "destination.pdf" |
| 86 | + expected_size = int(entry["object"]["metadata"]["size"]) |
| 87 | + downloaded = 0 |
| 88 | + with http.stream("GET", entry["pdf_url"]) as response: |
| 89 | + response.raise_for_status() |
| 90 | + with source.open("wb") as handle: |
| 91 | + for chunk in response.iter_bytes(1024 * 1024): |
| 92 | + downloaded += len(chunk) |
| 93 | + if downloaded > expected_size: |
| 94 | + raise ValueError("Source exceeds inventoried size") |
| 95 | + handle.write(chunk) |
| 96 | + with source.open("rb") as handle: |
| 97 | + if handle.read(5) != b"%PDF-": |
| 98 | + raise ValueError("Source does not have a PDF header") |
| 99 | + if downloaded != expected_size: |
| 100 | + raise ValueError("Source size differs from inventory") |
| 101 | + source_sha = digest(source) |
| 102 | + after = lookup_source(source_client, source_bucket, key) |
| 103 | + if after is None or fingerprint(after) != fingerprint(before): |
| 104 | + raise ValueError("Source changed during download") |
| 105 | + blob = bucket.get_blob(key) |
| 106 | + created = blob is None |
| 107 | + if created: |
| 108 | + blob = bucket.blob(key, chunk_size=8 * 1024 * 1024) |
| 109 | + blob.cache_control = "no-cache" |
| 110 | + blob.content_disposition = "inline" |
| 111 | + blob.metadata = {"researchos-source-sha256": source_sha} |
| 112 | + blob.upload_from_filename(str(source), content_type="application/pdf", |
| 113 | + if_generation_match=0, checksum="auto", timeout=120) |
| 114 | + generation = int(blob.generation) |
| 115 | + # Pin the read to the exact generation; never verify a different version. |
| 116 | + bucket.blob(key, generation=generation).download_to_filename( |
| 117 | + str(destination), if_generation_match=generation, checksum="auto", timeout=120) |
| 118 | + if destination.stat().st_size != downloaded or digest(destination) != source_sha: |
| 119 | + raise ValueError("Destination checksum/size mismatch; existing object was not overwritten") |
| 120 | + return {"paper_id": entry["id"], "source_key": key, |
| 121 | + "original_url": entry["pdf_url"], "source_fingerprint": fingerprint(after), |
| 122 | + "destination_bucket": bucket.name, "destination_key": key, |
| 123 | + "destination_generation": generation, "bytes": downloaded, |
| 124 | + "sha256": source_sha, "created": created, "status": "copied_verified"} |
| 125 | + |
| 126 | + |
| 127 | +def main(): |
| 128 | + root = Path(__file__).resolve().parents[1] |
| 129 | + parser = argparse.ArgumentParser(description=__doc__) |
| 130 | + parser.add_argument("--inventory", type=Path, default=root / "data/migration/paper-inventory.json") |
| 131 | + parser.add_argument("--env-file", type=Path, default=root / ".env") |
| 132 | + parser.add_argument("--project", default="researchos-513104") |
| 133 | + parser.add_argument("--bucket", default="researchos-papers") |
| 134 | + parser.add_argument("--limit", type=int, default=3, help="Default: small/median/large pilot; 0: all referenced PDFs") |
| 135 | + parser.add_argument("--journal", type=Path, default=root / "data/migration/copy-journal.jsonl") |
| 136 | + parser.add_argument("--execute", action="store_true", help="Actually copy; otherwise show selection only") |
| 137 | + parser.add_argument("--workers", type=int, default=1, help="Concurrent transfers (1-8)") |
| 138 | + args = parser.parse_args() |
| 139 | + if args.limit < 0: |
| 140 | + parser.error("--limit must be nonnegative") |
| 141 | + if not 1 <= args.workers <= 8: |
| 142 | + parser.error('--workers must be between 1 and 8') |
| 143 | + report = json.loads(args.inventory.read_text(encoding="utf-8-sig")) |
| 144 | + entries = select_entries(report, args.limit) |
| 145 | + print(json.dumps({"mode": "copy-and-verify" if args.execute else "dry-run", |
| 146 | + "bucket": args.bucket, "selected": len(entries), |
| 147 | + "bytes": sum(int(e["object"]["metadata"]["size"]) for e in entries)}, indent=2), flush=True) |
| 148 | + if not args.execute: |
| 149 | + return 0 |
| 150 | + from dotenv import load_dotenv |
| 151 | + from google.cloud import storage |
| 152 | + import httpx |
| 153 | + from supabase import create_client |
| 154 | + |
| 155 | + load_dotenv(args.env_file, override=False) |
| 156 | + base_url = os.environ.get("SUPABASE_URL", "").rstrip("/") |
| 157 | + secret = os.environ.get("SUPABASE_KEY", "") |
| 158 | + if not base_url.startswith("https://") or not secret: |
| 159 | + parser.error("Configure SUPABASE_URL and SUPABASE_KEY locally") |
| 160 | + source = create_client(base_url, secret) |
| 161 | + target = storage.Client(project=args.project).bucket(args.bucket) |
| 162 | + args.journal.parent.mkdir(parents=True, exist_ok=True) |
| 163 | + previous = {} |
| 164 | + if args.journal.exists(): |
| 165 | + for line in args.journal.read_text(encoding='utf-8').splitlines(): |
| 166 | + record = json.loads(line) |
| 167 | + previous[record['source_key']] = record |
| 168 | + failed = False |
| 169 | + with httpx.Client(timeout=120, follow_redirects=False, headers={"Accept-Encoding": "identity"}) as http, ThreadPoolExecutor(max_workers=args.workers) as pool: |
| 170 | + futures = {pool.submit(copy_one, entry, target, http, source, report['source_bucket'], base_url, |
| 171 | + previous.get(entry['source_key'])): entry for entry in entries} |
| 172 | + for future in as_completed(futures): |
| 173 | + entry = futures[future] |
| 174 | + if future.cancelled(): |
| 175 | + continue |
| 176 | + try: |
| 177 | + record = future.result() |
| 178 | + except Exception as error: |
| 179 | + # Avoid logging exception messages that can contain credential-bearing URLs. |
| 180 | + record = {"source_key": entry["source_key"], "status": "failed", |
| 181 | + "error_type": type(error).__name__} |
| 182 | + if isinstance(error, ValueError): |
| 183 | + record["detail"] = str(error) |
| 184 | + record["recorded_at"] = datetime.now(timezone.utc).isoformat() |
| 185 | + with args.journal.open("a", encoding="utf-8") as handle: |
| 186 | + handle.write(json.dumps(record) + "\n") |
| 187 | + handle.flush() |
| 188 | + os.fsync(handle.fileno()) |
| 189 | + print(json.dumps({key: value for key, value in record.items() |
| 190 | + if key in ("source_key", "status", "bytes", "created", "error_type", "detail")}), flush=True) |
| 191 | + if record["status"] == "failed": |
| 192 | + failed = True |
| 193 | + for pending in futures: |
| 194 | + pending.cancel() |
| 195 | + return 1 if failed else 0 |
| 196 | + |
| 197 | + |
| 198 | +if __name__ == "__main__": |
| 199 | + raise SystemExit(main()) |
0 commit comments