|
| 1 | +"""Preflight and journal verified PDF URL updates, with guarded rollback.""" |
| 2 | +from __future__ import annotations |
| 3 | + |
| 4 | +import argparse |
| 5 | +from datetime import datetime, timezone |
| 6 | +import json |
| 7 | +import os |
| 8 | +from pathlib import Path |
| 9 | +from urllib.parse import quote |
| 10 | + |
| 11 | +from copy_paper_storage import fingerprint |
| 12 | +from inventory_paper_storage import list_objects, list_papers, source_key |
| 13 | + |
| 14 | + |
| 15 | +def build_plan(report, verified, base_url, bucket): |
| 16 | + objects = {obj['key']: obj for obj in report['objects']} |
| 17 | + plan = [] |
| 18 | + for entry in report['managed_references']: |
| 19 | + key = entry['source_key'] |
| 20 | + record = verified.get(key) |
| 21 | + if source_key(entry['pdf_url'], base_url, report['source_bucket']) != key: |
| 22 | + raise ValueError('Inventory URL does not match the configured source') |
| 23 | + if not record or record['status'] != 'copied_verified': |
| 24 | + raise ValueError(f'No verified copy: {key}') |
| 25 | + if (record['destination_bucket'] != bucket or record['destination_key'] != key |
| 26 | + or record['original_url'] != entry['pdf_url'] |
| 27 | + or tuple(record['source_fingerprint']) != fingerprint(objects[key])): |
| 28 | + raise ValueError(f'Copy does not match inventory: {key}') |
| 29 | + plan.append({ |
| 30 | + 'paper_id': entry['id'], 'old_url': entry['pdf_url'], |
| 31 | + 'new_url': f'https://storage.googleapis.com/{bucket}/{quote(key, safe="/")}', |
| 32 | + 'key': key, 'generation': record['destination_generation'], |
| 33 | + 'bytes': record['bytes'], 'sha256': record['sha256'], |
| 34 | + 'source_fingerprint': record['source_fingerprint'], |
| 35 | + }) |
| 36 | + if not plan: |
| 37 | + raise ValueError('No managed references found') |
| 38 | + if report.get('missing_sources') or report.get('shared_source_keys'): |
| 39 | + raise ValueError('Resolve missing/shared source references before cutover') |
| 40 | + return plan |
| 41 | + |
| 42 | + |
| 43 | +def validate_plan(plan, current_papers, current_objects, destination_objects, rollback=False): |
| 44 | + papers = {row['id']: row for row in current_papers} |
| 45 | + sources = {obj['key']: obj for obj in current_objects} |
| 46 | + for entry in plan: |
| 47 | + row = papers.get(entry['paper_id']) |
| 48 | + expected, desired = (entry['new_url'], entry['old_url']) if rollback else (entry['old_url'], entry['new_url']) |
| 49 | + if row is None or row.get('pdf_url') not in (expected, desired): |
| 50 | + raise ValueError(f'Paper changed or deleted: {entry["paper_id"]}') |
| 51 | + source = sources.get(entry['key']) |
| 52 | + if source is None or fingerprint(source) != tuple(entry['source_fingerprint']): |
| 53 | + raise ValueError(f'Source changed or deleted: {entry["key"]}') |
| 54 | + destination = destination_objects.get(entry['key']) |
| 55 | + if (destination is None or int(destination.generation) != int(entry['generation']) |
| 56 | + or int(destination.size) != entry['bytes'] |
| 57 | + or (destination.metadata or {}).get('researchos-source-sha256') != entry['sha256']): |
| 58 | + raise ValueError(f'Destination changed or unverified: {entry["key"]}') |
| 59 | + |
| 60 | + |
| 61 | +def append_record(path, record): |
| 62 | + with path.open('a', encoding='utf-8') as handle: |
| 63 | + handle.write(json.dumps({**record, 'recorded_at': datetime.now(timezone.utc).isoformat()}) + '\n') |
| 64 | + handle.flush() |
| 65 | + os.fsync(handle.fileno()) |
| 66 | + |
| 67 | + |
| 68 | +def main(): |
| 69 | + root = Path(__file__).resolve().parents[1] |
| 70 | + folder = root / 'data/migration' |
| 71 | + parser = argparse.ArgumentParser(description=__doc__) |
| 72 | + parser.add_argument('--inventory', type=Path, default=folder / 'pre-copy-inventory.json') |
| 73 | + parser.add_argument('--copy-journal', type=Path, default=folder / 'copy-journal.jsonl') |
| 74 | + parser.add_argument('--backup', type=Path, default=folder / 'url-cutover-backup.json') |
| 75 | + parser.add_argument('--journal', type=Path, default=folder / 'url-cutover-journal.jsonl') |
| 76 | + parser.add_argument('--project', default='researchos-513104') |
| 77 | + parser.add_argument('--bucket', default='researchos-papers') |
| 78 | + parser.add_argument('--rollback', action='store_true') |
| 79 | + parser.add_argument('--execute', action='store_true') |
| 80 | + args = parser.parse_args() |
| 81 | + from dotenv import load_dotenv |
| 82 | + from google.cloud import storage |
| 83 | + from supabase import create_client |
| 84 | + |
| 85 | + load_dotenv(root / '.env', override=False) |
| 86 | + base_url = os.environ['SUPABASE_URL'].rstrip('/') |
| 87 | + db = create_client(base_url, os.environ['SUPABASE_KEY']) |
| 88 | + if args.backup.exists(): |
| 89 | + backup = json.loads(args.backup.read_text(encoding='utf-8')) |
| 90 | + if backup['bucket'] != args.bucket or backup['source_url'] != base_url: |
| 91 | + raise ValueError('Backup belongs to a different source or destination') |
| 92 | + plan = backup['plan'] |
| 93 | + source_bucket = backup['source_bucket'] |
| 94 | + else: |
| 95 | + if args.rollback: |
| 96 | + parser.error('Rollback requires an existing cutover backup') |
| 97 | + report = json.loads(args.inventory.read_text(encoding='utf-8')) |
| 98 | + verified = {} |
| 99 | + for line in args.copy_journal.read_text(encoding='utf-8').splitlines(): |
| 100 | + record = json.loads(line) |
| 101 | + verified[record['source_key']] = record |
| 102 | + plan = build_plan(report, verified, base_url, args.bucket) |
| 103 | + source_bucket = report['source_bucket'] |
| 104 | + papers = list_papers(db) |
| 105 | + source_objects = list_objects(db, source_bucket) |
| 106 | + destinations = {obj.name: obj for obj in storage.Client(project=args.project).list_blobs(args.bucket)} |
| 107 | + validate_plan(plan, papers, source_objects, destinations, args.rollback) |
| 108 | + # Detect new source references that were not included in the transfer plan. |
| 109 | + planned_ids = {entry['paper_id'] for entry in plan} |
| 110 | + if any(row['id'] not in planned_ids and source_key(row.get('pdf_url'), base_url, source_bucket) |
| 111 | + for row in papers): |
| 112 | + raise ValueError('New source references appeared; reconcile inventory before cutover') |
| 113 | + print(json.dumps({'mode': 'rollback' if args.rollback else 'apply-urls', |
| 114 | + 'execute': args.execute, 'verified_plan_rows': len(plan)}, indent=2), flush=True) |
| 115 | + if not args.execute: |
| 116 | + return 0 |
| 117 | + if not (folder / 'maintenance.flag').exists(): |
| 118 | + parser.error('Pause local application writes with backend/data/migration/maintenance.flag first') |
| 119 | + if not args.backup.exists(): |
| 120 | + rows = [] |
| 121 | + while True: |
| 122 | + page = db.table('papers').select('*').order('id').range(len(rows), len(rows) + 499).execute().data |
| 123 | + if not page: |
| 124 | + break |
| 125 | + rows.extend(page) |
| 126 | + with args.backup.open('x', encoding='utf-8') as handle: |
| 127 | + json.dump({'source_url': base_url, 'source_bucket': source_bucket, 'bucket': args.bucket, |
| 128 | + 'created_at': datetime.now(timezone.utc).isoformat(), |
| 129 | + 'plan': plan, 'paper_rows_before': rows}, handle, indent=2) |
| 130 | + handle.flush() |
| 131 | + os.fsync(handle.fileno()) |
| 132 | + count = 0 |
| 133 | + for entry in plan: |
| 134 | + old, new = (entry['new_url'], entry['old_url']) if args.rollback else (entry['old_url'], entry['new_url']) |
| 135 | + append_record(args.journal, {'paper_id': entry['paper_id'], 'old_url': old, |
| 136 | + 'new_url': new, 'status': 'pending'}) |
| 137 | + updated = db.table('papers').update({'pdf_url': new}).eq('id', entry['paper_id']).eq('pdf_url', old).execute().data |
| 138 | + if not updated: |
| 139 | + current = db.table('papers').select('pdf_url').eq('id', entry['paper_id']).execute().data |
| 140 | + if not current or current[0]['pdf_url'] != new: |
| 141 | + raise ValueError(f'Concurrent URL conflict: {entry["paper_id"]}') |
| 142 | + append_record(args.journal, {'paper_id': entry['paper_id'], 'old_url': old, |
| 143 | + 'new_url': new, 'status': 'confirmed'}) |
| 144 | + count += 1 |
| 145 | + if count % 25 == 0 or count == len(plan): |
| 146 | + print(f'Confirmed {count}/{len(plan)} URL updates', flush=True) |
| 147 | + after = list_papers(db) |
| 148 | + desired_by_id = {entry['paper_id']: entry['old_url'] if args.rollback else entry['new_url'] for entry in plan} |
| 149 | + for row in after: |
| 150 | + if row['id'] in desired_by_id and row.get('pdf_url') != desired_by_id[row['id']]: |
| 151 | + raise ValueError('Post-cutover URL verification failed') |
| 152 | + if {row['id'] for row in after} != {row['id'] for row in papers}: |
| 153 | + raise ValueError('Paper IDs changed during cutover') |
| 154 | + print(f'URL verification passed; {len(after)} paper IDs preserved.', flush=True) |
| 155 | + return 0 |
| 156 | + |
| 157 | + |
| 158 | +if __name__ == '__main__': |
| 159 | + raise SystemExit(main()) |
0 commit comments