Skip to content

Commit 12d7dfc

Browse files
feat(migration): add guarded PDF cutover and cleanup tools
Back up paper rows and journal compare-and-set URL changes with guarded rollback. Gate source cleanup on the recovery window and verified references, and provide explicit live GCS smoke checks using temporary records.
1 parent 570ae7e commit 12d7dfc

5 files changed

Lines changed: 416 additions & 0 deletions

File tree

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
"""Review or remove verified superseded Supabase PDFs after the recovery window."""
2+
import argparse
3+
from datetime import datetime, timedelta, timezone
4+
import json
5+
import os
6+
from pathlib import Path
7+
8+
from copy_paper_storage import fingerprint
9+
from inventory_paper_storage import list_objects, list_papers
10+
from switch_paper_urls import append_record
11+
12+
13+
def eligible_sources(plan, papers, sources, destinations):
14+
by_id = {row['id']: row for row in papers}
15+
active_urls = {row.get('pdf_url') for row in papers}
16+
eligible, skipped = [], []
17+
for entry in plan:
18+
key = entry['key']
19+
if key not in sources:
20+
continue
21+
row, dest = by_id.get(entry['paper_id']), destinations.get(key)
22+
if (row is None or row.get('pdf_url') != entry['new_url'] or entry['old_url'] in active_urls
23+
or fingerprint(sources[key]) != tuple(entry['source_fingerprint'])
24+
or dest is None or int(dest.generation) != entry['generation']
25+
or int(dest.size) != entry['bytes']
26+
or (dest.metadata or {}).get('researchos-source-sha256') != entry['sha256']):
27+
skipped.append(key)
28+
else:
29+
eligible.append(key)
30+
return eligible, skipped
31+
32+
33+
def main():
34+
root = Path(__file__).resolve().parents[1]
35+
folder = root / 'data/migration'
36+
parser = argparse.ArgumentParser(description=__doc__)
37+
parser.add_argument('--backup', type=Path, default=folder / 'url-cutover-backup.json')
38+
parser.add_argument('--execute', action='store_true')
39+
args = parser.parse_args()
40+
from dotenv import load_dotenv
41+
from google.cloud import storage
42+
from supabase import create_client
43+
44+
backup = json.loads(args.backup.read_text(encoding='utf-8'))
45+
load_dotenv(root / '.env', override=False)
46+
if os.environ['SUPABASE_URL'].rstrip('/') != backup['source_url']:
47+
raise ValueError('Backup belongs to a different Supabase project')
48+
if args.execute and datetime.now(timezone.utc) < datetime.fromisoformat(backup['created_at']) + timedelta(days=7):
49+
parser.error('The seven-day recovery window has not elapsed; dry-run only for now')
50+
if args.execute and not (folder / 'maintenance.flag').exists():
51+
parser.error('Pause local writes with backend/data/migration/maintenance.flag before source cleanup')
52+
db = create_client(backup['source_url'], os.environ['SUPABASE_KEY'])
53+
papers = list_papers(db)
54+
sources = {obj['key']: obj for obj in list_objects(db, backup['source_bucket'])}
55+
destinations = {obj.name: obj for obj in storage.Client(project=os.environ.get('GCS_PROJECT_ID')).list_blobs(backup['bucket'])}
56+
eligible, skipped = eligible_sources(backup['plan'], papers, sources, destinations)
57+
print(json.dumps({'execute': args.execute, 'eligible_objects': len(eligible),
58+
'eligible_bytes': sum(int(sources[key]['metadata']['size']) for key in eligible),
59+
'skipped_for_review': skipped,
60+
'recovery_window_ends': (datetime.fromisoformat(backup['created_at']) + timedelta(days=7)).isoformat()}, indent=2))
61+
if args.execute:
62+
for key in eligible:
63+
append_record(folder / 'source-cleanup-journal.jsonl', {'key': key, 'status': 'pending'})
64+
db.storage.from_(backup['source_bucket']).remove([key])
65+
append_record(folder / 'source-cleanup-journal.jsonl', {'key': key, 'status': 'removed'})
66+
print('Source cleanup complete. Unreferenced and changed objects were not deleted.')
67+
68+
69+
if __name__ == '__main__':
70+
main()
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
"""Explicit live integration check using temporary papers, cleaned up on exit."""
2+
import argparse
3+
import json
4+
import os
5+
from pathlib import Path
6+
import sys
7+
8+
9+
def main():
10+
parser = argparse.ArgumentParser(description=__doc__)
11+
parser.add_argument('--execute', action='store_true')
12+
args = parser.parse_args()
13+
if not args.execute:
14+
print('Use --execute to create temporary validation papers and GCS objects, then remove them.')
15+
return
16+
root = Path(__file__).resolve().parents[1]
17+
sys.path.insert(0, str(root))
18+
from dotenv import load_dotenv
19+
load_dotenv(root / '.env', override=False)
20+
os.environ.update(PAPER_STORAGE_PROVIDER='gcs', GCS_PROJECT_ID='researchos-513104',
21+
GCS_PDF_BUCKET='researchos-papers')
22+
import httpx
23+
import pymupdf
24+
from fastapi.testclient import TestClient
25+
from app import app
26+
from models.paper import PaperCreate
27+
from services import paper_service, pdf_service, pdf_text_service
28+
from services.db import get_client
29+
30+
def make_pdf(text):
31+
doc = pymupdf.open()
32+
page = doc.new_page()
33+
page.insert_text((72, 72), text)
34+
result = doc.tobytes()
35+
doc.close()
36+
return result
37+
38+
# Do not enter the context manager: avoid unrelated startup seeding writes.
39+
client = TestClient(app)
40+
ids = []
41+
checks = []
42+
try:
43+
paper = paper_service.create_paper(PaperCreate(
44+
title='Temporary GCS migration validation', authors=[], year=2026, venue=''))
45+
ids.append(paper.id)
46+
first = make_pdf('ResearchOS GCS first validation document')
47+
response = client.post(f'/api/papers/{paper.id}/pdf', files={'file': ('test.pdf', first, 'application/pdf')})
48+
response.raise_for_status()
49+
assert response.json()['hasStoredPdf'] is True
50+
url = response.json()['pdfUrl']
51+
assert url.startswith('https://storage.googleapis.com/researchos-papers/')
52+
checks.append('API upload and managed-PDF flag')
53+
with httpx.Client(timeout=60) as http:
54+
response = http.get(url, headers={'Origin': 'http://localhost:5173'})
55+
response.raise_for_status()
56+
assert response.content == first
57+
assert response.headers.get('access-control-allow-origin') == 'http://localhost:5173'
58+
assert 'application/pdf' in response.headers.get('content-type', '')
59+
assert 'inline' in response.headers.get('content-disposition', '')
60+
response = http.get(url, headers={'Range': 'bytes=0-4', 'Origin': 'http://localhost:5173'})
61+
assert response.status_code == 206 and response.content == b'%PDF-'
62+
checks.append('Anonymous download, CORS, PDF metadata, and Range requests')
63+
assert client.post(f'/api/papers/{paper.id}/pdf/fetch').status_code == 422
64+
checks.append('Already-stored GCS fetch rejected')
65+
extracted = pdf_text_service.extract_and_cache(paper.id, url)
66+
assert 'first validation' in extracted['markdown']
67+
checks.append('Cache-miss PDF text extraction')
68+
replacement = make_pdf('ResearchOS GCS replacement validation document')
69+
response = client.post(f'/api/papers/{paper.id}/pdf', files={'file': ('test.pdf', replacement, 'application/pdf')})
70+
response.raise_for_status()
71+
assert pdf_text_service.get_cached_text(paper.id) is None
72+
with httpx.Client(timeout=60) as http:
73+
assert http.get(url).content == replacement
74+
extracted = pdf_text_service.extract_and_cache(paper.id, url)
75+
assert 'replacement validation' in extracted['markdown']
76+
checks.append('Atomic replacement and extracted-text cache invalidation')
77+
response = client.delete(f'/api/papers/{paper.id}/pdf')
78+
assert response.status_code == 204
79+
assert paper_service.get_paper(paper.id).pdf_url is None
80+
assert pdf_text_service.get_cached_text(paper.id) is None
81+
assert pdf_service.get_gcs_client().bucket('researchos-papers').get_blob(f'{paper.id}.pdf') is None
82+
checks.append('PDF removal clears storage, reference, and text cache')
83+
client.post(f'/api/papers/{paper.id}/pdf', files={'file': ('test.pdf', first, 'application/pdf')}).raise_for_status()
84+
assert client.delete(f'/api/papers/{paper.id}').status_code == 204
85+
assert paper_service.get_paper(paper.id) is None
86+
assert pdf_service.get_gcs_client().bucket('researchos-papers').get_blob(f'{paper.id}.pdf') is None
87+
checks.append('Whole-paper deletion removes managed PDF')
88+
second = paper_service.create_paper(PaperCreate(
89+
title='Temporary GCS bulk deletion validation', authors=[], year=2026, venue=''))
90+
ids.append(second.id)
91+
client.post(f'/api/papers/{second.id}/pdf', files={'file': ('test.pdf', first, 'application/pdf')}).raise_for_status()
92+
response = client.post('/api/batch/items', json={'items': [{'id': second.id, 'itemType': 'paper'}], 'action': 'delete'})
93+
response.raise_for_status()
94+
assert response.json()['succeededIds'] == [second.id]
95+
assert pdf_service.get_gcs_client().bucket('researchos-papers').get_blob(f'{second.id}.pdf') is None
96+
checks.append('Bulk paper deletion removes managed PDF')
97+
print(json.dumps({'checks_passed': checks}, indent=2))
98+
finally:
99+
for paper_id in ids:
100+
url = f'https://storage.googleapis.com/researchos-papers/{paper_id}.pdf'
101+
pdf_service.delete_pdf(paper_id, url)
102+
pdf_text_service.delete_cached_text(paper_id)
103+
get_client().table('papers').delete().eq('id', paper_id).execute()
104+
client.close()
105+
print('Temporary validation records and live GCS objects removed.')
106+
107+
108+
if __name__ == '__main__':
109+
main()
Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
"""Preflight and journal verified PDF URL updates, with guarded rollback."""
2+
from __future__ import annotations
3+
4+
import argparse
5+
from datetime import datetime, timezone
6+
import json
7+
import os
8+
from pathlib import Path
9+
from urllib.parse import quote
10+
11+
from copy_paper_storage import fingerprint
12+
from inventory_paper_storage import list_objects, list_papers, source_key
13+
14+
15+
def build_plan(report, verified, base_url, bucket):
16+
objects = {obj['key']: obj for obj in report['objects']}
17+
plan = []
18+
for entry in report['managed_references']:
19+
key = entry['source_key']
20+
record = verified.get(key)
21+
if source_key(entry['pdf_url'], base_url, report['source_bucket']) != key:
22+
raise ValueError('Inventory URL does not match the configured source')
23+
if not record or record['status'] != 'copied_verified':
24+
raise ValueError(f'No verified copy: {key}')
25+
if (record['destination_bucket'] != bucket or record['destination_key'] != key
26+
or record['original_url'] != entry['pdf_url']
27+
or tuple(record['source_fingerprint']) != fingerprint(objects[key])):
28+
raise ValueError(f'Copy does not match inventory: {key}')
29+
plan.append({
30+
'paper_id': entry['id'], 'old_url': entry['pdf_url'],
31+
'new_url': f'https://storage.googleapis.com/{bucket}/{quote(key, safe="/")}',
32+
'key': key, 'generation': record['destination_generation'],
33+
'bytes': record['bytes'], 'sha256': record['sha256'],
34+
'source_fingerprint': record['source_fingerprint'],
35+
})
36+
if not plan:
37+
raise ValueError('No managed references found')
38+
if report.get('missing_sources') or report.get('shared_source_keys'):
39+
raise ValueError('Resolve missing/shared source references before cutover')
40+
return plan
41+
42+
43+
def validate_plan(plan, current_papers, current_objects, destination_objects, rollback=False):
44+
papers = {row['id']: row for row in current_papers}
45+
sources = {obj['key']: obj for obj in current_objects}
46+
for entry in plan:
47+
row = papers.get(entry['paper_id'])
48+
expected, desired = (entry['new_url'], entry['old_url']) if rollback else (entry['old_url'], entry['new_url'])
49+
if row is None or row.get('pdf_url') not in (expected, desired):
50+
raise ValueError(f'Paper changed or deleted: {entry["paper_id"]}')
51+
source = sources.get(entry['key'])
52+
if source is None or fingerprint(source) != tuple(entry['source_fingerprint']):
53+
raise ValueError(f'Source changed or deleted: {entry["key"]}')
54+
destination = destination_objects.get(entry['key'])
55+
if (destination is None or int(destination.generation) != int(entry['generation'])
56+
or int(destination.size) != entry['bytes']
57+
or (destination.metadata or {}).get('researchos-source-sha256') != entry['sha256']):
58+
raise ValueError(f'Destination changed or unverified: {entry["key"]}')
59+
60+
61+
def append_record(path, record):
62+
with path.open('a', encoding='utf-8') as handle:
63+
handle.write(json.dumps({**record, 'recorded_at': datetime.now(timezone.utc).isoformat()}) + '\n')
64+
handle.flush()
65+
os.fsync(handle.fileno())
66+
67+
68+
def main():
69+
root = Path(__file__).resolve().parents[1]
70+
folder = root / 'data/migration'
71+
parser = argparse.ArgumentParser(description=__doc__)
72+
parser.add_argument('--inventory', type=Path, default=folder / 'pre-copy-inventory.json')
73+
parser.add_argument('--copy-journal', type=Path, default=folder / 'copy-journal.jsonl')
74+
parser.add_argument('--backup', type=Path, default=folder / 'url-cutover-backup.json')
75+
parser.add_argument('--journal', type=Path, default=folder / 'url-cutover-journal.jsonl')
76+
parser.add_argument('--project', default='researchos-513104')
77+
parser.add_argument('--bucket', default='researchos-papers')
78+
parser.add_argument('--rollback', action='store_true')
79+
parser.add_argument('--execute', action='store_true')
80+
args = parser.parse_args()
81+
from dotenv import load_dotenv
82+
from google.cloud import storage
83+
from supabase import create_client
84+
85+
load_dotenv(root / '.env', override=False)
86+
base_url = os.environ['SUPABASE_URL'].rstrip('/')
87+
db = create_client(base_url, os.environ['SUPABASE_KEY'])
88+
if args.backup.exists():
89+
backup = json.loads(args.backup.read_text(encoding='utf-8'))
90+
if backup['bucket'] != args.bucket or backup['source_url'] != base_url:
91+
raise ValueError('Backup belongs to a different source or destination')
92+
plan = backup['plan']
93+
source_bucket = backup['source_bucket']
94+
else:
95+
if args.rollback:
96+
parser.error('Rollback requires an existing cutover backup')
97+
report = json.loads(args.inventory.read_text(encoding='utf-8'))
98+
verified = {}
99+
for line in args.copy_journal.read_text(encoding='utf-8').splitlines():
100+
record = json.loads(line)
101+
verified[record['source_key']] = record
102+
plan = build_plan(report, verified, base_url, args.bucket)
103+
source_bucket = report['source_bucket']
104+
papers = list_papers(db)
105+
source_objects = list_objects(db, source_bucket)
106+
destinations = {obj.name: obj for obj in storage.Client(project=args.project).list_blobs(args.bucket)}
107+
validate_plan(plan, papers, source_objects, destinations, args.rollback)
108+
# Detect new source references that were not included in the transfer plan.
109+
planned_ids = {entry['paper_id'] for entry in plan}
110+
if any(row['id'] not in planned_ids and source_key(row.get('pdf_url'), base_url, source_bucket)
111+
for row in papers):
112+
raise ValueError('New source references appeared; reconcile inventory before cutover')
113+
print(json.dumps({'mode': 'rollback' if args.rollback else 'apply-urls',
114+
'execute': args.execute, 'verified_plan_rows': len(plan)}, indent=2), flush=True)
115+
if not args.execute:
116+
return 0
117+
if not (folder / 'maintenance.flag').exists():
118+
parser.error('Pause local application writes with backend/data/migration/maintenance.flag first')
119+
if not args.backup.exists():
120+
rows = []
121+
while True:
122+
page = db.table('papers').select('*').order('id').range(len(rows), len(rows) + 499).execute().data
123+
if not page:
124+
break
125+
rows.extend(page)
126+
with args.backup.open('x', encoding='utf-8') as handle:
127+
json.dump({'source_url': base_url, 'source_bucket': source_bucket, 'bucket': args.bucket,
128+
'created_at': datetime.now(timezone.utc).isoformat(),
129+
'plan': plan, 'paper_rows_before': rows}, handle, indent=2)
130+
handle.flush()
131+
os.fsync(handle.fileno())
132+
count = 0
133+
for entry in plan:
134+
old, new = (entry['new_url'], entry['old_url']) if args.rollback else (entry['old_url'], entry['new_url'])
135+
append_record(args.journal, {'paper_id': entry['paper_id'], 'old_url': old,
136+
'new_url': new, 'status': 'pending'})
137+
updated = db.table('papers').update({'pdf_url': new}).eq('id', entry['paper_id']).eq('pdf_url', old).execute().data
138+
if not updated:
139+
current = db.table('papers').select('pdf_url').eq('id', entry['paper_id']).execute().data
140+
if not current or current[0]['pdf_url'] != new:
141+
raise ValueError(f'Concurrent URL conflict: {entry["paper_id"]}')
142+
append_record(args.journal, {'paper_id': entry['paper_id'], 'old_url': old,
143+
'new_url': new, 'status': 'confirmed'})
144+
count += 1
145+
if count % 25 == 0 or count == len(plan):
146+
print(f'Confirmed {count}/{len(plan)} URL updates', flush=True)
147+
after = list_papers(db)
148+
desired_by_id = {entry['paper_id']: entry['old_url'] if args.rollback else entry['new_url'] for entry in plan}
149+
for row in after:
150+
if row['id'] in desired_by_id and row.get('pdf_url') != desired_by_id[row['id']]:
151+
raise ValueError('Post-cutover URL verification failed')
152+
if {row['id'] for row in after} != {row['id'] for row in papers}:
153+
raise ValueError('Paper IDs changed during cutover')
154+
print(f'URL verification passed; {len(after)} paper IDs preserved.', flush=True)
155+
return 0
156+
157+
158+
if __name__ == '__main__':
159+
raise SystemExit(main())
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
from types import SimpleNamespace
2+
import unittest
3+
4+
from cleanup_source_pdfs import eligible_sources
5+
6+
7+
class CleanupTests(unittest.TestCase):
8+
def setUp(self):
9+
self.plan = [{'paper_id': 'p_1', 'key': 'p_1.pdf', 'old_url': 'source', 'new_url': 'dest',
10+
'generation': 123, 'bytes': 10, 'sha256': 'hash',
11+
'source_fingerprint': ['id', 'v1', None, 10, None]}]
12+
self.papers = [{'id': 'p_1', 'pdf_url': 'dest'}]
13+
self.sources = {'p_1.pdf': {'id': 'id', 'version': 'v1', 'metadata': {'size': 10}},
14+
'orphan.pdf': {'id': 'unrelated', 'metadata': {'size': 20}}}
15+
self.destinations = {'p_1.pdf': SimpleNamespace(generation=123, size=10,
16+
metadata={'researchos-source-sha256': 'hash'})}
17+
18+
def result(self):
19+
return eligible_sources(self.plan, self.papers, self.sources, self.destinations)
20+
21+
def test_only_verified_manifest_objects_are_eligible(self):
22+
self.assertEqual(self.result(), (['p_1.pdf'], []))
23+
24+
def test_source_still_referenced_by_any_paper_is_retained(self):
25+
self.papers.append({'id': 'p_2', 'pdf_url': 'source'})
26+
self.assertEqual(self.result(), ([], ['p_1.pdf']))
27+
28+
def test_changed_destination_is_retained_for_review(self):
29+
self.destinations['p_1.pdf'].generation = 124
30+
self.assertEqual(self.result(), ([], ['p_1.pdf']))
31+
32+
def test_changed_source_is_retained_for_review(self):
33+
self.sources['p_1.pdf']['version'] = 'v2'
34+
self.assertEqual(self.result(), ([], ['p_1.pdf']))
35+
36+
37+
if __name__ == '__main__':
38+
unittest.main()

0 commit comments

Comments
 (0)