Skip to content

Commit 8ee9e09

Browse files
rustyconoverclaude
andcommitted
catalog: carry a format branch's reader options on the wire
ScanBranch.format_options was a declared, documented field with no wire column. Both ends were built and neither was connected: a worker could set it, and the C++ rewriter has always consumed it as the reader's named arguments — but nothing carried it between them, so the map arrived permanently empty. Because the rewriter ASSIGNS the reader's named arguments from it rather than merging, that made it impossible to pass a format branch's reader a single option. A CSV branch could name its files and could not say they were pipe-delimited. No error; the reader just ran with defaults. Encoded as a 1-row IPC batch whose column names are the option names — the same shape the arguments blob uses, and for the same reason: an option value may be any Arrow type, so no static schema could describe the map. Also fixes ScanBranch.deserialize, which dropped format_name and format_locations entirely, so a deserialized format branch came back looking like a malformed function branch. Invisible only because the C++ client is the sole consumer and parses the columns itself. Fixture: multi_branch_format, exercised by the new catalog test. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 798fc31 commit 8ee9e09

4 files changed

Lines changed: 84 additions & 0 deletions

File tree

0 Bytes
Binary file not shown.

‎tests/test_wire_record_schema_parity.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ def _scan_branch() -> ScanBranch:
9797
source_table="events",
9898
format_name="parquet",
9999
format_locations=["s3://bucket/a.parquet"],
100+
format_options={"delim": pa.scalar("|", pa.string())},
100101
)
101102

102103

‎vgi/_test_fixtures/worker.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,15 @@ def _build_fixture_stats() -> tuple[
966966
columns=schema(n=pa.int64()),
967967
comment="Multi-branch: sequence(50) + read_parquet — used by multi_branch_heterogeneous.test",
968968
),
969+
# Format branch: the worker names a FORMAT and locations, and the
970+
# client resolves the reader. Its format_options carry the reader's
971+
# named arguments — the file is pipe-delimited with a header, and
972+
# read_csv gets both columns only if those options arrive.
973+
Table(
974+
name="multi_branch_format",
975+
columns=schema(n=pa.int64(), label=pa.string()),
976+
comment="Format branch: read_csv with delim/header options — used by multi_branch_format.test",
977+
),
969978
# Iceberg cold arm: one VGI sequence() hot arm + one native
970979
# iceberg_scan arm. The iceberg table is created by the test at a
971980
# well-known path via COPY … TO (FORMAT iceberg) (see
@@ -1638,6 +1647,44 @@ def table_scan_branches_get(
16381647
required_extensions=[],
16391648
)
16401649

1650+
# multi_branch_format: a FORMAT branch — the worker names a format and
1651+
# some locations, and the client resolves the reader (read_csv) without
1652+
# the worker knowing that reader's argument spelling.
1653+
#
1654+
# The options are the point. A format branch's format_options BECOME the
1655+
# reader's named arguments, and for a long time nothing carried them on
1656+
# the wire, so a format branch could not pass a single reader option.
1657+
# The file is pipe-delimited with a header, but DuckDB's sniffer works
1658+
# both of those out unaided — verified by mutation, the test passed with
1659+
# the options discarded. So it also sends `nullstr`, which cannot be
1660+
# inferred from the data: exactly one label becomes NULL, and nothing
1661+
# else about the result changes.
1662+
if schema_name.lower() == "data" and name.lower() == "multi_branch_format":
1663+
return ScanBranchesResult(
1664+
branches=[
1665+
ScanBranch(
1666+
function_name="",
1667+
positional_arguments=[],
1668+
named_arguments={},
1669+
format_name="csv",
1670+
format_locations=[f"{_BRANCH_DIR}/vgi_format_branch.csv"],
1671+
format_options={
1672+
"delim": pa.scalar("|", pa.string()),
1673+
"header": pa.scalar(True, pa.bool_()),
1674+
# The load-bearing one. DuckDB's CSV sniffer infers
1675+
# the delimiter and the header by itself, so those two
1676+
# prove nothing on their own — the test passed with
1677+
# the options thrown away. `nullstr` cannot be
1678+
# guessed: it turns one specific label into NULL, so
1679+
# the row count stays 5 either way and only the
1680+
# non-null count moves.
1681+
"nullstr": pa.scalar("row_2", pa.string()),
1682+
},
1683+
),
1684+
],
1685+
required_extensions=[],
1686+
)
1687+
16411688
# multi_branch_iceberg: one VGI arm (sequence(50)) + one native
16421689
# iceberg_scan arm pointing at a well-known iceberg table dir the test
16431690
# creates via COPY … TO (FORMAT iceberg) before querying. The iceberg

‎vgi/catalog/catalog_interface.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1011,6 +1011,29 @@ def scan_arguments_from(
10111011
# relevant methods below.
10121012

10131013

1014+
def _serialize_named_scalars(values: "dict[str, pa.Scalar]") -> bytes | None: # type: ignore[type-arg]
1015+
"""Pack name→scalar pairs into a 1-row IPC batch, or ``None`` when empty.
1016+
1017+
The column NAMES carry the keys, which is the same trick ``arguments`` uses
1018+
and for the same reason: a value may be any Arrow type, so no static schema
1019+
could describe the map. ``None`` rather than an empty batch so "no options"
1020+
costs no bytes and reads as absent rather than as an empty struct.
1021+
"""
1022+
if not values:
1023+
return None
1024+
schema = pa.schema([pa.field(name, value.type) for name, value in values.items()])
1025+
batch = pa.RecordBatch.from_pylist([dict(values)], schema=schema)
1026+
return serialize_record_batch_bytes(batch)
1027+
1028+
1029+
def _deserialize_named_scalars(data: bytes | None) -> "dict[str, pa.Scalar]": # type: ignore[type-arg]
1030+
"""Inverse of :func:`_serialize_named_scalars`. Absent or empty reads as ``{}``."""
1031+
if not data:
1032+
return {}
1033+
batch, _ = deserialize_record_batch(data)
1034+
return {field.name: batch.column(field.name)[0] for field in batch.schema}
1035+
1036+
10141037
@dataclass(frozen=True)
10151038
class ScanBranch:
10161039
"""One physical source backing a multi-branch scan.
@@ -1098,6 +1121,11 @@ class ScanBranch:
10981121
pa.field("source_table", pa.string(), nullable=True),
10991122
pa.field("format_name", pa.string(), nullable=True),
11001123
pa.field("format_locations", pa.list_(pa.string()), nullable=True),
1124+
# Reader options as a 1-row IPC batch whose COLUMN NAMES are the
1125+
# option names — the same nested-batch trick `arguments` uses,
1126+
# because an option value can be any Arrow type and no static schema
1127+
# could express that.
1128+
pa.field("format_options", pa.binary(), nullable=True),
11011129
] # type: ignore[arg-type]
11021130
)
11031131

@@ -1126,6 +1154,7 @@ def to_row_dict(self) -> dict[str, Any]:
11261154
"writable": self.writable,
11271155
"format_name": self.format_name,
11281156
"format_locations": list(self.format_locations),
1157+
"format_options": _serialize_named_scalars(self.format_options),
11291158
"source_catalog": self.source_catalog,
11301159
"source_schema": self.source_schema,
11311160
"source_table": self.source_table,
@@ -1173,6 +1202,13 @@ def deserialize(cls, batch: pa.RecordBatch) -> Self:
11731202
source_catalog=cast("str | None", row.get("source_catalog")),
11741203
source_schema=cast("str | None", row.get("source_schema")),
11751204
source_table=cast("str | None", row.get("source_table")),
1205+
# The format fields round-trip too. They were omitted here, so a
1206+
# deserialized format branch came back looking like a (malformed)
1207+
# function branch — invisible in practice only because the C++ client
1208+
# is the sole consumer and parses the columns itself.
1209+
format_name=cast("str | None", row.get("format_name")),
1210+
format_locations=list(cast("list[str] | None", row.get("format_locations")) or []),
1211+
format_options=_deserialize_named_scalars(cast("bytes | None", row.get("format_options"))),
11761212
)
11771213

11781214

0 commit comments

Comments
 (0)