Skip to content

Commit 50953cf

Browse files
rustyconoverclaude
andcommitted
tests: dynamic-filter state across HTTP turns
Drives the real dynamic_filter_echo fixture through the in-process HTTP stack, playing the client with hand-built tick deltas, and pins: a turn parses a bounded number of deltas however many ticks came before; a resent revision is stale and accumulates nothing; a compacted history rebuilds the predicate order the worker had (remove + re-add); a tombstone outlives compaction. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 0283898 commit 50953cf

1 file changed

Lines changed: 199 additions & 0 deletions

File tree

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
# Copyright 2026 Query Farm LLC - https://query.farm
2+
3+
"""Dynamic-filter state across HTTP turns: bounded replay, exact reconstruction.
4+
5+
An HTTP producer rebuilds its filters from the call and cursor tokens on every
6+
turn. The cursor used to carry every tick's delta and replay all of them, each
7+
re-validated through the embedded DuckDB evaluator, so turn ``k`` did ``k``
8+
parses: a 100-batch Top-N scan spent ~20 s there. These tests drive the real
9+
``dynamic_filter_echo`` fixture through the real HTTP stack, playing the client
10+
(the requests are hand-built; every response is the worker's), and pin both
11+
halves of the fix: a turn's parse work stays constant as ticks accumulate, and
12+
the state rebuilt from the compacted history is the state the worker had.
13+
"""
14+
15+
from __future__ import annotations
16+
17+
import base64
18+
import contextlib
19+
import json
20+
from collections.abc import Iterator
21+
from typing import Any
22+
23+
import pyarrow as pa
24+
import pytest
25+
from vgi_rpc.rpc import RpcServer
26+
27+
from vgi import filter_v2
28+
from vgi._test_fixtures.worker import ExampleWorker
29+
from vgi.arguments import Arguments
30+
from vgi.invocation import FunctionType
31+
from vgi.protocol import BindRequest, InitRequest, VgiProtocol
32+
33+
_KEY = b"\x5a" * 32
34+
_METADATA: dict[bytes | str, bytes | str] = {
35+
b"vgi_filter_encoding": b"vgi.filters.v2",
36+
b"vgi_filter_version": b"2",
37+
b"vgi_evaluation_context": b"vgi.none.v1",
38+
}
39+
40+
41+
def _document_batch(document: dict[str, object], *values: int) -> pa.RecordBatch:
42+
fields: list[pa.Field[Any]] = [pa.field("filter_spec", pa.string(), nullable=False)]
43+
arrays: list[pa.Array[Any]] = [pa.array([json.dumps(document, separators=(",", ":"))])]
44+
for index, value in enumerate(values):
45+
fields.append(pa.field(f"value_{index}", pa.int64()))
46+
arrays.append(pa.array([value], type=pa.int64()))
47+
return pa.RecordBatch.from_arrays(arrays, schema=pa.schema(fields, metadata=_METADATA))
48+
49+
50+
def _empty_snapshot() -> pa.RecordBatch:
51+
return _document_batch(
52+
{"encoding": "vgi.filters.v2", "semantics": "vgi.duckdb.standard.v1", "kind": "snapshot", "predicates": []}
53+
)
54+
55+
56+
def _upsert(predicate_id: str, revision: int, op: str, value_ref: int) -> dict[str, object]:
57+
return {
58+
"operation": "upsert",
59+
"id": predicate_id,
60+
"revision": revision,
61+
"mode": "advisory",
62+
"source": "top_n",
63+
"expression": {
64+
"node": "comparison",
65+
"op": op,
66+
"left": {"node": "column_ref", "column_index": 0, "column_name": "n"},
67+
"right": {"node": "literal", "value_ref": value_ref},
68+
},
69+
}
70+
71+
72+
def _remove(predicate_id: str, revision: int) -> dict[str, object]:
73+
return {"operation": "remove", "id": predicate_id, "revision": revision}
74+
75+
76+
def _tick(updates: list[dict[str, object]], *values: int) -> pa.KeyValueMetadata:
77+
"""Tick metadata carrying one delta, framed the way the C++ client frames it."""
78+
batch = _document_batch(
79+
{"encoding": "vgi.filters.v2", "semantics": "vgi.duckdb.standard.v1", "kind": "delta", "updates": updates},
80+
*values,
81+
)
82+
sink = pa.BufferOutputStream()
83+
with pa.ipc.new_stream(sink, batch.schema) as writer:
84+
writer.write_batch(batch)
85+
return pa.KeyValueMetadata({b"vgi_pushdown_filters": base64.b64encode(sink.getvalue().to_pybytes())})
86+
87+
88+
@contextlib.contextmanager
89+
def _echo_stream(count: int, batch_size: int) -> Iterator[Any]:
90+
"""Open a ``dynamic_filter_echo`` scan over the in-process HTTP stack."""
91+
from vgi_rpc.http import http_connect, make_sync_client
92+
93+
worker = ExampleWorker(quiet=True)
94+
worker._signing_key = _KEY
95+
client = make_sync_client(RpcServer(VgiProtocol, worker, enable_describe=False), token_key=_KEY)
96+
bind = BindRequest(
97+
function_name="dynamic_filter_echo",
98+
arguments=Arguments(positional=(pa.scalar(count),), named={"batch_size": pa.scalar(batch_size)}),
99+
function_type=FunctionType.TABLE,
100+
input_schema=None,
101+
)
102+
with http_connect(VgiProtocol, client=client, compression_level=None) as proxy: # type: ignore[type-abstract]
103+
resp = proxy.bind(request=bind)
104+
stream = proxy.init(
105+
request=InitRequest(
106+
bind_call=bind,
107+
output_schema=resp.output_schema,
108+
bind_opaque_data=resp.opaque_data,
109+
pushdown_filters=_empty_snapshot(),
110+
)
111+
)
112+
try:
113+
yield stream
114+
finally:
115+
stream.close()
116+
117+
118+
@pytest.fixture
119+
def parse_count(monkeypatch: pytest.MonkeyPatch) -> list[int]:
120+
"""Count delta documents the worker parses (a spy: it records, never answers)."""
121+
calls = [0]
122+
original = filter_v2._Parser.parse_delta
123+
124+
def counting(self: Any) -> Any:
125+
calls[0] += 1
126+
return original(self)
127+
128+
monkeypatch.setattr(filter_v2._Parser, "parse_delta", counting)
129+
return calls
130+
131+
132+
def test_a_turn_parses_a_bounded_number_of_deltas(parse_count: list[int]) -> None:
133+
"""A tightening Top-N bound sends a delta every tick; no turn replays them all.
134+
135+
Rows descend from 999 in batches of 10 and each tick narrows ``n < bound``,
136+
so every tick's delta changes the live predicate -- the case the C++ client
137+
cannot skip. Before compaction turn ``k`` parsed ``k`` deltas.
138+
"""
139+
ticks = 60
140+
with _echo_stream(count=1000, batch_size=10) as stream:
141+
first = stream.tick() # the init turn's batch; metadata rides continuations only
142+
assert first.batch.num_rows == 10
143+
per_turn: list[int] = []
144+
for revision in range(1, ticks + 1):
145+
bound = 1000 - 10 * revision # rows of this tick are [bound - 10, bound - 1]
146+
before = parse_count[0]
147+
batch = stream.tick(_tick([_upsert("top_n:0", revision, "lt", 0)], bound)).batch
148+
per_turn.append(parse_count[0] - before)
149+
assert batch.column("n").to_pylist() == list(range(bound - 1, bound - 11, -1))
150+
assert batch.column("pushed_filters")[0].as_py().count("ConstantFilter(n < ") == 1
151+
assert f"ConstantFilter(n < {bound})" in batch.column("pushed_filters")[0].as_py()
152+
153+
# One parse for the new delta, one to replay the single delta the cursor keeps.
154+
assert max(per_turn) <= 2, per_turn
155+
156+
157+
def test_a_resent_revision_is_stale_and_adds_no_replay(parse_count: list[int]) -> None:
158+
"""Resending a revision with another value is a stale no-op: nothing applies, nothing accumulates."""
159+
with _echo_stream(count=1000, batch_size=10) as stream:
160+
stream.tick()
161+
stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 100000))
162+
before = parse_count[0]
163+
for _ in range(20):
164+
batch = stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 5)).batch
165+
assert batch.num_rows == 10 # n < 5 would have emptied it
166+
assert "ConstantFilter(n < 100000)" in batch.column("pushed_filters")[0].as_py()
167+
assert parse_count[0] - before <= 2 * 20
168+
169+
170+
def test_rebuilt_state_keeps_the_order_the_worker_had() -> None:
171+
"""A removed-then-re-added predicate keeps its position across a turn boundary.
172+
173+
Deltas: {a:1, b:1} -> [a, b]; {remove a:2, b:1} -> [b]; {a:3, b:1} -> [b, a].
174+
The compacted history is the deltas that first carried a:3 and b:1 -- the
175+
third and the first -- and replaying those alone yields [a, b]. The turn
176+
after the third delta has to show the order the third turn itself had.
177+
"""
178+
with _echo_stream(count=1000, batch_size=10) as stream:
179+
stream.tick()
180+
stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0), _upsert("top_n:1", 1, "gt", 1)], 100000, 5))
181+
stream.tick(_tick([_remove("top_n:0", 2), _upsert("top_n:1", 1, "gt", 0)], 5))
182+
applied = stream.tick(_tick([_upsert("top_n:0", 3, "lt", 0), _upsert("top_n:1", 1, "gt", 1)], 99999, 5)).batch
183+
rebuilt = stream.tick().batch # no delta: filters come purely from the tokens
184+
assert applied.num_rows == rebuilt.num_rows == 10
185+
live = applied.column("pushed_filters")[0].as_py()
186+
assert live.index("ConstantFilter(n > 5)") < live.index("ConstantFilter(n < 99999)"), live
187+
assert rebuilt.column("pushed_filters")[0].as_py() == live
188+
189+
190+
def test_a_tombstone_survives_compaction() -> None:
191+
"""A removal stays in force after its upsert is compacted away: a stale upsert cannot resurrect it."""
192+
with _echo_stream(count=1000, batch_size=10) as stream:
193+
stream.tick()
194+
stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 100000))
195+
stream.tick(_tick([_remove("top_n:0", 2)]))
196+
stream.tick() # a turn rebuilt from the compacted tokens
197+
stale = stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 5)).batch
198+
assert stale.column("pushed_filters")[0].as_py() == "(none)"
199+
assert stale.num_rows == 10

0 commit comments

Comments
 (0)