|
| 1 | +"""Load a SQLite target database into Postgres 16. |
| 2 | +
|
| 3 | +Makes the "Postgres 16 + StackExchange-mini" claim real: it copies a shipped |
| 4 | +SQLite database (Chinook, or any BIRD Mini-Dev slice — ``codebase_community`` is |
| 5 | +the StackExchange-derived one) table-by-table into a Postgres schema the pipeline |
| 6 | +can then query through its read-only connection. |
| 7 | +
|
| 8 | +Run against the docker-compose Postgres (see docker-compose.yml, profile |
| 9 | +``postgres``). Loading requires WRITE privileges, so pass the OWNER DSN here |
| 10 | +(``postgres`` superuser), NOT the ``nl_sql_ro`` role the pipeline uses at query |
| 11 | +time: |
| 12 | +
|
| 13 | + docker compose --profile postgres up -d |
| 14 | + uv run python scripts/load_postgres.py \ |
| 15 | + --sqlite data/bird_mini_dev/MINIDEV/dev_databases/codebase_community/codebase_community.sqlite \ |
| 16 | + --pg-dsn postgresql://postgres:postgres@localhost:5433/nl_sql_demo |
| 17 | +
|
| 18 | +The tables are created with ``if_exists=replace`` (idempotent re-runs). Type |
| 19 | +mapping is delegated to pandas/SQLAlchemy, which is sufficient for these demo |
| 20 | +databases; this is a load tool, not a general SQLite→Postgres migrator. |
| 21 | +""" |
| 22 | + |
| 23 | +from __future__ import annotations |
| 24 | + |
| 25 | +import argparse |
| 26 | +import sqlite3 |
| 27 | +import sys |
| 28 | +from pathlib import Path |
| 29 | + |
| 30 | +import pandas as pd |
| 31 | +from sqlalchemy import Engine, create_engine, text |
| 32 | + |
| 33 | +from nl_sql.db.connection import _normalise_pg_driver |
| 34 | + |
| 35 | + |
| 36 | +def _sqlite_tables(sqlite_path: Path) -> list[str]: |
| 37 | + con = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) |
| 38 | + try: |
| 39 | + rows = con.execute( |
| 40 | + "SELECT name FROM sqlite_master WHERE type = 'table' " |
| 41 | + "AND name NOT LIKE 'sqlite_%' ORDER BY name" |
| 42 | + ).fetchall() |
| 43 | + finally: |
| 44 | + con.close() |
| 45 | + return [r[0] for r in rows] |
| 46 | + |
| 47 | + |
| 48 | +def load_sqlite_into_postgres( |
| 49 | + sqlite_path: Path, |
| 50 | + pg_dsn: str, |
| 51 | + *, |
| 52 | + only_tables: list[str] | None = None, |
| 53 | + chunk_size: int = 5_000, |
| 54 | +) -> dict[str, int]: |
| 55 | + """Copy every table from the SQLite file into Postgres. Returns row counts.""" |
| 56 | + if not sqlite_path.exists(): |
| 57 | + raise FileNotFoundError(f"SQLite database not found: {sqlite_path}") |
| 58 | + |
| 59 | + tables = only_tables or _sqlite_tables(sqlite_path) |
| 60 | + src = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) |
| 61 | + dst = create_engine(_normalise_pg_driver(pg_dsn), future=True) |
| 62 | + loaded: dict[str, int] = {} |
| 63 | + try: |
| 64 | + for table in tables: |
| 65 | + total = 0 |
| 66 | + first = True |
| 67 | + for frame in pd.read_sql_query(f'SELECT * FROM "{table}"', src, chunksize=chunk_size): |
| 68 | + frame.to_sql( |
| 69 | + table, |
| 70 | + dst, |
| 71 | + if_exists="replace" if first else "append", |
| 72 | + index=False, |
| 73 | + method="multi", |
| 74 | + ) |
| 75 | + total += len(frame) |
| 76 | + first = False |
| 77 | + if first: |
| 78 | + # Table was empty — still create it so the schema matches. |
| 79 | + pd.read_sql_query(f'SELECT * FROM "{table}" LIMIT 0', src).to_sql( |
| 80 | + table, dst, if_exists="replace", index=False |
| 81 | + ) |
| 82 | + loaded[table] = total |
| 83 | + print(f" {table}: {total} rows", flush=True) |
| 84 | + _grant_readonly(dst) |
| 85 | + finally: |
| 86 | + src.close() |
| 87 | + dst.dispose() |
| 88 | + return loaded |
| 89 | + |
| 90 | + |
| 91 | +def _grant_readonly(engine: Engine) -> None: |
| 92 | + """Best-effort: grant SELECT to nl_sql_ro if that role exists. |
| 93 | +
|
| 94 | + The role is created by scripts/sql/postgres_init.sql on first container boot. |
| 95 | + Newly created tables are not covered by the role's default privileges until |
| 96 | + granted, so re-grant after a load. Silently skipped if the role is absent. |
| 97 | + """ |
| 98 | + with engine.begin() as conn: |
| 99 | + exists = conn.execute(text("SELECT 1 FROM pg_roles WHERE rolname = 'nl_sql_ro'")).fetchone() |
| 100 | + if exists: |
| 101 | + conn.execute(text("GRANT SELECT ON ALL TABLES IN SCHEMA public TO nl_sql_ro")) |
| 102 | + |
| 103 | + |
| 104 | +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: |
| 105 | + parser = argparse.ArgumentParser(description="Load a SQLite DB into Postgres.") |
| 106 | + parser.add_argument("--sqlite", required=True, type=Path, help="path to the .sqlite file") |
| 107 | + parser.add_argument("--pg-dsn", required=True, help="owner DSN, e.g. postgresql://postgres:...") |
| 108 | + parser.add_argument("--table", action="append", dest="tables", help="limit to table (repeat)") |
| 109 | + parser.add_argument("--chunk", type=int, default=5_000, help="rows per insert batch") |
| 110 | + return parser.parse_args(argv) |
| 111 | + |
| 112 | + |
| 113 | +def main(argv: list[str] | None = None) -> int: |
| 114 | + args = _parse_args(argv) |
| 115 | + print(f"Loading {args.sqlite} -> Postgres", flush=True) |
| 116 | + counts = load_sqlite_into_postgres( |
| 117 | + args.sqlite, args.pg_dsn, only_tables=args.tables, chunk_size=args.chunk |
| 118 | + ) |
| 119 | + print(f"Done: {len(counts)} tables, {sum(counts.values())} rows total", flush=True) |
| 120 | + return 0 |
| 121 | + |
| 122 | + |
| 123 | +if __name__ == "__main__": |
| 124 | + sys.exit(main()) |
0 commit comments