Skip to content

Commit e40a262

Browse files
committed
feat(client): parent_row provenance for blended row-transform functions
Adds vgi.protocol._decode_parent_rows, the client-side counterpart to the existing _merge_parent_rows encoder — decodes vgi_rpc.parent_row#b64 off an exchange output batch's metadata, byte-for-byte matching the DuckDB C++ extension's own DecodeParentRow reference decoder (identity-map fallback when row counts match, exact byte-length check, per-value [0, input_rows) range check). Threads it through Client.table_in_out_function's exchange loop (_process_batch_on_worker -> _worker_thread_loop -> _distribute_and_collect) behind a new parent_row_callback parameter, mirroring table_function's existing batch_metadata_callback contract. Backward-compat is the load-bearing constraint: decoding only activates when a caller passes the callback, so every existing (non-blended) table-in-out call is unaffected — proven by the existing 82-test tests/table_in_out/ suite passing unchanged plus a dedicated test_no_callback_no_decode_no_raise. Also adds _combine_parent_rows, concatenating per-sub-batch parent-row lists in emission order for the HAVE_MORE_OUTPUT multi-round case (slice 2 of the plan, folded in here since it was cheap and removes a "assume one batch" landmine rather than deferring it). New tests/table_in_out/test_blended_exchange.py exercises the blended fixtures (blended_explode, hostile_provenance, geo_encode, row_sum) at the RPC level for the first time — prior coverage was resolve_metadata()-only. Covers exact parent-row pairing (not just the output row set), 1->0 filtering, the identity-map path, and all three HostileProvenanceFunction adversarial payloads on both subprocess and HTTP transports. Bug found and fixed during testing: the initial _decode_parent_rows checked row-count-mismatch before the output_rows==0 case, so a fully-filtered (1->0) blended emit was misclassified as a worker bug — reordered so a 0-row output is always unambiguous, since _merge_parent_rows never encodes provenance for an empty emit in the first place. First slice of the blended row-transform plan (~/.claude/plans/okay-make-a-new-hashed-aurora.md) — vgi-python side only; vgi-polars's _row_transform.py bridge lands separately. Full suite: 2506 passed, 101 skipped (1 pre-existing, unrelated mypy type:ignore lint failure deselected — confirmed present on main before this change).
1 parent 8312394 commit e40a262

3 files changed

Lines changed: 421 additions & 13 deletions

File tree

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
# Copyright 2025, 2026 Query Farm LLC - https://query.farm
2+
3+
"""RPC-level exchange coverage for blended (`RowTransformFunction`) table functions.
4+
5+
`test_blended_metadata.py` covers `resolve_metadata()` in isolation — no worker
6+
subprocess. This file is the first exchange-level coverage for the blended
7+
fixtures registered in `vgi/_test_fixtures/table_in_out.py`: it drives them
8+
through `Client.table_in_out_function(...)`'s new `parent_row_callback`, which
9+
decodes `vgi_rpc.parent_row#b64` provenance for a 1->N / 1->0 emit (see
10+
`vgi.protocol._decode_parent_rows`).
11+
"""
12+
13+
from __future__ import annotations
14+
15+
from typing import Any
16+
17+
import pyarrow as pa
18+
import pytest
19+
20+
from tests.conftest import make_schema
21+
from vgi.arguments import Arguments
22+
from vgi.client import Client
23+
from vgi.client.client import ClientError
24+
25+
26+
class TestBlendedExplodeProvenance:
27+
"""`blended_explode`: 1->N fan-out with per-output-row parent_row provenance."""
28+
29+
def test_parent_row_pairing_is_correct(self, fixture_worker: str) -> None:
30+
"""Each output row's parent index must point at the input row that produced it.
31+
32+
Asserting only the final row *set* would pass even for a naive
33+
"assume identity" implementation whenever fan-out counts differ
34+
across rows -- so this reconstructs the exact expected
35+
(parent_row, emitted_value) pairs in emission order and compares.
36+
"""
37+
input_schema = make_schema([pa.field("n", pa.int64())])
38+
counts = [0, 1, 3, 2]
39+
batch = pa.RecordBatch.from_pydict({"n": counts}, schema=input_schema)
40+
41+
parent_rows_seen: list[int] = []
42+
43+
with Client(fixture_worker) as client:
44+
results = list(
45+
client.table_in_out_function(
46+
function_name="blended_explode",
47+
schema_name="main",
48+
input=iter([batch]),
49+
parent_row_callback=parent_rows_seen.extend,
50+
)
51+
)
52+
53+
assert len(results) == 1
54+
out = results[0]
55+
i_values = out.column("i").to_pylist()
56+
57+
assert len(i_values) == sum(counts)
58+
assert len(parent_rows_seen) == len(i_values)
59+
60+
expected_pairs = [(row_idx, k) for row_idx, n in enumerate(counts) for k in range(n)]
61+
actual_pairs = list(zip(parent_rows_seen, i_values, strict=True))
62+
assert actual_pairs == expected_pairs
63+
64+
def test_all_rows_filtered_to_empty_result(self, fixture_worker: str) -> None:
65+
"""n=0 for every input row -> a correctly-schemed, zero-row output."""
66+
input_schema = make_schema([pa.field("n", pa.int64())])
67+
batch = pa.RecordBatch.from_pydict({"n": [0, 0, 0]}, schema=input_schema)
68+
69+
parent_rows_seen: list[list[int]] = []
70+
71+
with Client(fixture_worker) as client:
72+
results = list(
73+
client.table_in_out_function(
74+
function_name="blended_explode",
75+
schema_name="main",
76+
input=iter([batch]),
77+
parent_row_callback=parent_rows_seen.append,
78+
)
79+
)
80+
81+
assert len(results) == 1
82+
out = results[0]
83+
assert out.num_rows == 0
84+
assert out.schema.names == ["i"]
85+
assert parent_rows_seen == [[]]
86+
87+
def test_identity_map_when_row_counts_match(self, fixture_worker: str) -> None:
88+
"""n=1 for every row -> a 1:1 emit; the identity map is a valid decode.
89+
90+
`blended_explode` never encodes `vgi_rpc.parent_row` for a 1:1 emit
91+
(the C++ reference assumes identity for that case too), so this
92+
exercises `_decode_parent_rows`'s absent-metadata branch, not just
93+
the explicit-array branch the other tests exercise.
94+
"""
95+
input_schema = make_schema([pa.field("n", pa.int64())])
96+
batch = pa.RecordBatch.from_pydict({"n": [1, 1, 1]}, schema=input_schema)
97+
98+
parent_rows_seen: list[int] = []
99+
100+
with Client(fixture_worker) as client:
101+
results = list(
102+
client.table_in_out_function(
103+
function_name="blended_explode",
104+
schema_name="main",
105+
input=iter([batch]),
106+
parent_row_callback=parent_rows_seen.extend,
107+
)
108+
)
109+
110+
assert len(results) == 1
111+
assert results[0].column("i").to_pylist() == [0, 0, 0]
112+
assert parent_rows_seen == [0, 1, 2]
113+
114+
115+
class TestHostileProvenance:
116+
"""`hostile_provenance`: adversarial worker payloads must raise, never corrupt."""
117+
118+
@pytest.mark.parametrize("mode", ["range", "length", "base64"])
119+
def test_malformed_provenance_raises_client_error(
120+
self,
121+
client_transport: Any,
122+
mode: str,
123+
) -> None:
124+
"""Each hostile mode must raise ClientError on both subprocess and HTTP.
125+
126+
Symmetry matters here: an asymmetry between transports would mean one
127+
parse path validates and the other doesn't -- a security bug, not
128+
just an inconsistency.
129+
"""
130+
input_schema = make_schema([pa.field("x", pa.int64())])
131+
batch = pa.RecordBatch.from_pydict({"x": [1, 2, 3]}, schema=input_schema)
132+
133+
with client_transport() as client, pytest.raises(ClientError):
134+
list(
135+
client.table_in_out_function(
136+
function_name="hostile_provenance",
137+
schema_name="main",
138+
input=iter([batch]),
139+
arguments=Arguments(named={"mode": pa.scalar(mode)}),
140+
parent_row_callback=lambda _rows: None,
141+
)
142+
)
143+
144+
def test_no_callback_no_decode_no_raise(self, fixture_worker: str) -> None:
145+
"""Without `parent_row_callback`, malformed provenance is never even decoded.
146+
147+
`decode_parent_rows` is derived from `parent_row_callback is not None`
148+
-- this is the backward-compat guarantee that an ordinary (non-blended)
149+
table-in-out caller, which never passes the callback, is completely
150+
unaffected by this feature existing.
151+
"""
152+
input_schema = make_schema([pa.field("x", pa.int64())])
153+
batch = pa.RecordBatch.from_pydict({"x": [1, 2, 3]}, schema=input_schema)
154+
155+
with Client(fixture_worker) as client:
156+
results = list(
157+
client.table_in_out_function(
158+
function_name="hostile_provenance",
159+
schema_name="main",
160+
input=iter([batch]),
161+
arguments=Arguments(named={"mode": pa.scalar("range")}),
162+
)
163+
)
164+
165+
assert len(results) == 1
166+
assert results[0].column("hv").to_pylist() == [1, 2, 3]
167+
168+
169+
class TestBlendedArityAndVarargs:
170+
"""Broader coverage once the core provenance mechanism is proven."""
171+
172+
def test_geo_encode_two_arg_overload(self, fixture_worker: str) -> None:
173+
"""`geo_encode(lat, lon)` -- the 2-arg overload, identity 1:1 map."""
174+
input_schema = make_schema([pa.field("latitude", pa.float64()), pa.field("longitude", pa.float64())])
175+
batch = pa.RecordBatch.from_pydict(
176+
{"latitude": [52.0, 48.5], "longitude": [13.0, 2.3]},
177+
schema=input_schema,
178+
)
179+
180+
parent_rows_seen: list[int] = []
181+
182+
with Client(fixture_worker) as client:
183+
results = list(
184+
client.table_in_out_function(
185+
function_name="geo_encode",
186+
schema_name="main",
187+
input=iter([batch]),
188+
arguments=Arguments(named={"precision": pa.scalar(1)}),
189+
parent_row_callback=parent_rows_seen.extend,
190+
)
191+
)
192+
193+
assert len(results) == 1
194+
assert results[0].column("geohash").to_pylist() == ["52.0:13.0", "48.5:2.3"]
195+
assert parent_rows_seen == [0, 1]
196+
197+
def test_geo_encode_three_arg_overload(self, fixture_worker: str) -> None:
198+
"""`geo_encode(lat, lon, alt)` -- same Meta.name, resolved by input arity."""
199+
input_schema = make_schema(
200+
[
201+
pa.field("latitude", pa.float64()),
202+
pa.field("longitude", pa.float64()),
203+
pa.field("altitude", pa.float64()),
204+
]
205+
)
206+
batch = pa.RecordBatch.from_pydict(
207+
{"latitude": [52.0], "longitude": [13.0], "altitude": [100.0]},
208+
schema=input_schema,
209+
)
210+
211+
with Client(fixture_worker) as client:
212+
results = list(
213+
client.table_in_out_function(
214+
function_name="geo_encode",
215+
schema_name="main",
216+
input=iter([batch]),
217+
arguments=Arguments(named={"precision": pa.scalar(0)}),
218+
)
219+
)
220+
221+
assert len(results) == 1
222+
assert results[0].column("geohash").to_pylist() == ["52.0:13.0:100.0"]
223+
224+
def test_row_sum_varargs(self, fixture_worker: str) -> None:
225+
"""`row_sum(a, b, c)` -- varargs positional input, positionally-named columns."""
226+
input_schema = make_schema([pa.field("col0", pa.float64()), pa.field("col1", pa.float64())])
227+
batch = pa.RecordBatch.from_pydict({"col0": [1.0, -2.0], "col1": [2.0, -3.0]}, schema=input_schema)
228+
229+
with Client(fixture_worker) as client:
230+
results = list(
231+
client.table_in_out_function(
232+
function_name="row_sum",
233+
schema_name="main",
234+
input=iter([batch]),
235+
arguments=Arguments(named={"absolute": pa.scalar(True)}),
236+
)
237+
)
238+
239+
assert len(results) == 1
240+
assert results[0].column("row_sum").to_pylist() == [3.0, 5.0]

0 commit comments

Comments
 (0)