|
| 1 | +# Copyright 2026 Query Farm LLC - https://query.farm |
| 2 | + |
| 3 | +"""Dynamic-filter state across HTTP turns: bounded replay, exact reconstruction. |
| 4 | +
|
| 5 | +An HTTP producer rebuilds its filters from the call and cursor tokens on every |
| 6 | +turn. The cursor used to carry every tick's delta and replay all of them, each |
| 7 | +re-validated through the embedded DuckDB evaluator, so turn ``k`` did ``k`` |
| 8 | +parses: a 100-batch Top-N scan spent ~20 s there. These tests drive the real |
| 9 | +``dynamic_filter_echo`` fixture through the real HTTP stack, playing the client |
| 10 | +(the requests are hand-built; every response is the worker's), and pin both |
| 11 | +halves of the fix: a turn's parse work stays constant as ticks accumulate, and |
| 12 | +the state rebuilt from the compacted history is the state the worker had. |
| 13 | +""" |
| 14 | + |
| 15 | +from __future__ import annotations |
| 16 | + |
| 17 | +import base64 |
| 18 | +import contextlib |
| 19 | +import json |
| 20 | +from collections.abc import Iterator |
| 21 | +from typing import Any |
| 22 | + |
| 23 | +import pyarrow as pa |
| 24 | +import pytest |
| 25 | +from vgi_rpc.rpc import RpcServer |
| 26 | + |
| 27 | +from vgi import filter_v2 |
| 28 | +from vgi._test_fixtures.worker import ExampleWorker |
| 29 | +from vgi.arguments import Arguments |
| 30 | +from vgi.invocation import FunctionType |
| 31 | +from vgi.protocol import BindRequest, InitRequest, VgiProtocol |
| 32 | + |
| 33 | +_KEY = b"\x5a" * 32 |
| 34 | +_METADATA: dict[bytes | str, bytes | str] = { |
| 35 | + b"vgi_filter_encoding": b"vgi.filters.v2", |
| 36 | + b"vgi_filter_version": b"2", |
| 37 | + b"vgi_evaluation_context": b"vgi.none.v1", |
| 38 | +} |
| 39 | + |
| 40 | + |
| 41 | +def _document_batch(document: dict[str, object], *values: int) -> pa.RecordBatch: |
| 42 | + fields: list[pa.Field[Any]] = [pa.field("filter_spec", pa.string(), nullable=False)] |
| 43 | + arrays: list[pa.Array[Any]] = [pa.array([json.dumps(document, separators=(",", ":"))])] |
| 44 | + for index, value in enumerate(values): |
| 45 | + fields.append(pa.field(f"value_{index}", pa.int64())) |
| 46 | + arrays.append(pa.array([value], type=pa.int64())) |
| 47 | + return pa.RecordBatch.from_arrays(arrays, schema=pa.schema(fields, metadata=_METADATA)) |
| 48 | + |
| 49 | + |
| 50 | +def _empty_snapshot() -> pa.RecordBatch: |
| 51 | + return _document_batch( |
| 52 | + {"encoding": "vgi.filters.v2", "semantics": "vgi.duckdb.standard.v1", "kind": "snapshot", "predicates": []} |
| 53 | + ) |
| 54 | + |
| 55 | + |
| 56 | +def _upsert(predicate_id: str, revision: int, op: str, value_ref: int) -> dict[str, object]: |
| 57 | + return { |
| 58 | + "operation": "upsert", |
| 59 | + "id": predicate_id, |
| 60 | + "revision": revision, |
| 61 | + "mode": "advisory", |
| 62 | + "source": "top_n", |
| 63 | + "expression": { |
| 64 | + "node": "comparison", |
| 65 | + "op": op, |
| 66 | + "left": {"node": "column_ref", "column_index": 0, "column_name": "n"}, |
| 67 | + "right": {"node": "literal", "value_ref": value_ref}, |
| 68 | + }, |
| 69 | + } |
| 70 | + |
| 71 | + |
| 72 | +def _remove(predicate_id: str, revision: int) -> dict[str, object]: |
| 73 | + return {"operation": "remove", "id": predicate_id, "revision": revision} |
| 74 | + |
| 75 | + |
| 76 | +def _tick(updates: list[dict[str, object]], *values: int) -> pa.KeyValueMetadata: |
| 77 | + """Tick metadata carrying one delta, framed the way the C++ client frames it.""" |
| 78 | + batch = _document_batch( |
| 79 | + {"encoding": "vgi.filters.v2", "semantics": "vgi.duckdb.standard.v1", "kind": "delta", "updates": updates}, |
| 80 | + *values, |
| 81 | + ) |
| 82 | + sink = pa.BufferOutputStream() |
| 83 | + with pa.ipc.new_stream(sink, batch.schema) as writer: |
| 84 | + writer.write_batch(batch) |
| 85 | + return pa.KeyValueMetadata({b"vgi_pushdown_filters": base64.b64encode(sink.getvalue().to_pybytes())}) |
| 86 | + |
| 87 | + |
| 88 | +@contextlib.contextmanager |
| 89 | +def _echo_stream(count: int, batch_size: int) -> Iterator[Any]: |
| 90 | + """Open a ``dynamic_filter_echo`` scan over the in-process HTTP stack.""" |
| 91 | + from vgi_rpc.http import http_connect, make_sync_client |
| 92 | + |
| 93 | + worker = ExampleWorker(quiet=True) |
| 94 | + worker._signing_key = _KEY |
| 95 | + client = make_sync_client(RpcServer(VgiProtocol, worker, enable_describe=False), token_key=_KEY) |
| 96 | + bind = BindRequest( |
| 97 | + function_name="dynamic_filter_echo", |
| 98 | + arguments=Arguments(positional=(pa.scalar(count),), named={"batch_size": pa.scalar(batch_size)}), |
| 99 | + function_type=FunctionType.TABLE, |
| 100 | + input_schema=None, |
| 101 | + ) |
| 102 | + with http_connect(VgiProtocol, client=client, compression_level=None) as proxy: # type: ignore[type-abstract] |
| 103 | + resp = proxy.bind(request=bind) |
| 104 | + stream = proxy.init( |
| 105 | + request=InitRequest( |
| 106 | + bind_call=bind, |
| 107 | + output_schema=resp.output_schema, |
| 108 | + bind_opaque_data=resp.opaque_data, |
| 109 | + pushdown_filters=_empty_snapshot(), |
| 110 | + ) |
| 111 | + ) |
| 112 | + try: |
| 113 | + yield stream |
| 114 | + finally: |
| 115 | + stream.close() |
| 116 | + |
| 117 | + |
| 118 | +@pytest.fixture |
| 119 | +def parse_count(monkeypatch: pytest.MonkeyPatch) -> list[int]: |
| 120 | + """Count delta documents the worker parses (a spy: it records, never answers).""" |
| 121 | + calls = [0] |
| 122 | + original = filter_v2._Parser.parse_delta |
| 123 | + |
| 124 | + def counting(self: Any) -> Any: |
| 125 | + calls[0] += 1 |
| 126 | + return original(self) |
| 127 | + |
| 128 | + monkeypatch.setattr(filter_v2._Parser, "parse_delta", counting) |
| 129 | + return calls |
| 130 | + |
| 131 | + |
| 132 | +def test_a_turn_parses_a_bounded_number_of_deltas(parse_count: list[int]) -> None: |
| 133 | + """A tightening Top-N bound sends a delta every tick; no turn replays them all. |
| 134 | +
|
| 135 | + Rows descend from 999 in batches of 10 and each tick narrows ``n < bound``, |
| 136 | + so every tick's delta changes the live predicate -- the case the C++ client |
| 137 | + cannot skip. Before compaction turn ``k`` parsed ``k`` deltas. |
| 138 | + """ |
| 139 | + ticks = 60 |
| 140 | + with _echo_stream(count=1000, batch_size=10) as stream: |
| 141 | + first = stream.tick() # the init turn's batch; metadata rides continuations only |
| 142 | + assert first.batch.num_rows == 10 |
| 143 | + per_turn: list[int] = [] |
| 144 | + for revision in range(1, ticks + 1): |
| 145 | + bound = 1000 - 10 * revision # rows of this tick are [bound - 10, bound - 1] |
| 146 | + before = parse_count[0] |
| 147 | + batch = stream.tick(_tick([_upsert("top_n:0", revision, "lt", 0)], bound)).batch |
| 148 | + per_turn.append(parse_count[0] - before) |
| 149 | + assert batch.column("n").to_pylist() == list(range(bound - 1, bound - 11, -1)) |
| 150 | + assert batch.column("pushed_filters")[0].as_py().count("ConstantFilter(n < ") == 1 |
| 151 | + assert f"ConstantFilter(n < {bound})" in batch.column("pushed_filters")[0].as_py() |
| 152 | + |
| 153 | + # One parse for the new delta, one to replay the single delta the cursor keeps. |
| 154 | + assert max(per_turn) <= 2, per_turn |
| 155 | + |
| 156 | + |
| 157 | +def test_a_resent_revision_is_stale_and_adds_no_replay(parse_count: list[int]) -> None: |
| 158 | + """Resending a revision with another value is a stale no-op: nothing applies, nothing accumulates.""" |
| 159 | + with _echo_stream(count=1000, batch_size=10) as stream: |
| 160 | + stream.tick() |
| 161 | + stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 100000)) |
| 162 | + before = parse_count[0] |
| 163 | + for _ in range(20): |
| 164 | + batch = stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 5)).batch |
| 165 | + assert batch.num_rows == 10 # n < 5 would have emptied it |
| 166 | + assert "ConstantFilter(n < 100000)" in batch.column("pushed_filters")[0].as_py() |
| 167 | + assert parse_count[0] - before <= 2 * 20 |
| 168 | + |
| 169 | + |
| 170 | +def test_rebuilt_state_keeps_the_order_the_worker_had() -> None: |
| 171 | + """A removed-then-re-added predicate keeps its position across a turn boundary. |
| 172 | +
|
| 173 | + Deltas: {a:1, b:1} -> [a, b]; {remove a:2, b:1} -> [b]; {a:3, b:1} -> [b, a]. |
| 174 | + The compacted history is the deltas that first carried a:3 and b:1 -- the |
| 175 | + third and the first -- and replaying those alone yields [a, b]. The turn |
| 176 | + after the third delta has to show the order the third turn itself had. |
| 177 | + """ |
| 178 | + with _echo_stream(count=1000, batch_size=10) as stream: |
| 179 | + stream.tick() |
| 180 | + stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0), _upsert("top_n:1", 1, "gt", 1)], 100000, 5)) |
| 181 | + stream.tick(_tick([_remove("top_n:0", 2), _upsert("top_n:1", 1, "gt", 0)], 5)) |
| 182 | + applied = stream.tick(_tick([_upsert("top_n:0", 3, "lt", 0), _upsert("top_n:1", 1, "gt", 1)], 99999, 5)).batch |
| 183 | + rebuilt = stream.tick().batch # no delta: filters come purely from the tokens |
| 184 | + assert applied.num_rows == rebuilt.num_rows == 10 |
| 185 | + live = applied.column("pushed_filters")[0].as_py() |
| 186 | + assert live.index("ConstantFilter(n > 5)") < live.index("ConstantFilter(n < 99999)"), live |
| 187 | + assert rebuilt.column("pushed_filters")[0].as_py() == live |
| 188 | + |
| 189 | + |
| 190 | +def test_a_tombstone_survives_compaction() -> None: |
| 191 | + """A removal stays in force after its upsert is compacted away: a stale upsert cannot resurrect it.""" |
| 192 | + with _echo_stream(count=1000, batch_size=10) as stream: |
| 193 | + stream.tick() |
| 194 | + stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 100000)) |
| 195 | + stream.tick(_tick([_remove("top_n:0", 2)])) |
| 196 | + stream.tick() # a turn rebuilt from the compacted tokens |
| 197 | + stale = stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 5)).batch |
| 198 | + assert stale.column("pushed_filters")[0].as_py() == "(none)" |
| 199 | + assert stale.num_rows == 10 |
0 commit comments