Skip to content

Commit ced8edc

Browse files
fix(migrate): remove migration command to remote hf json
1 parent 86f832c commit ced8edc

5 files changed

Lines changed: 2 additions & 177 deletions

File tree

.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,3 @@ experiments/parquet-history/*_work/
2525
/qa/
2626

2727
CLAUDE.md
28-
experiments/migrate_work/

docs/rollups.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ Tags are stored one way in every queried store: a native list of `(key, value, c
5050
length_m)`. In DuckDB (the store and the rollup) that is a `LIST<STRUCT>`; in Postgres it is an array
5151
of the composite type `osmsg_tag`. A tag breakdown reads that column directly, the same code on either
5252
engine, with no per-query JSON parsing. The published `changefiles` dataset stores the same native
53-
list, so history reads it with a direct column copy. An existing Postgres deployment moves to the
54-
native column once with `docs/migrate_pg_native_tags.sql`; a fresh deployment creates it on first start.
53+
list, so history reads it with a direct column copy. A Postgres deployment creates the composite type
54+
and column on first start.
5555

5656
`osmsg.catalog` combines two sources into one relation, split at the frontier: history comes from the
5757
`hashtag_changeset` rollup, and the recent tail (from the frontier on) is derived on the fly from the

osmsg/maintain/cli.py

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -72,24 +72,6 @@ def convert_cmd(
7272
info(f"datasets written to {out}/changefiles and {out}/changesets")
7373

7474

75-
@maintain_app.command("migrate-tags")
76-
def migrate_tags_cmd(
77-
repo: Annotated[str, typer.Option("--repo", help="HuggingFace dataset repo id to migrate in place.")],
78-
work_dir: Annotated[Path, typer.Option("--work-dir", help="Scratch dir for converted partitions.")] = Path(
79-
"migrate_work"
80-
),
81-
memory_limit: Annotated[
82-
str, typer.Option("--memory-limit", help="DuckDB memory cap per partition (e.g. 12GB).")
83-
] = "12GB",
84-
) -> None:
85-
"""Rewrite the published changefiles partitions from JSON tag_stats to the native tags column
86-
(one-time, idempotent). Needs a HuggingFace login and uv/uvx for the upload."""
87-
from .migrate import migrate_changefiles_tags
88-
89-
migrated, skipped = migrate_changefiles_tags(repo, work_dir, memory_limit=memory_limit)
90-
info(f"migrate-tags: {migrated} migrated, {skipped} already native")
91-
92-
9375
@maintain_app.command("prune-pg")
9476
def prune_pg_cmd(
9577
psql_dsn: Annotated[str, typer.Option("--psql-dsn", help="Postgres DSN of the deployment to prune.")],

osmsg/maintain/migrate.py

Lines changed: 0 additions & 121 deletions
This file was deleted.

tests/test_migrate.py

Lines changed: 0 additions & 35 deletions
This file was deleted.

0 commit comments

Comments
 (0)