Skip to content

Commit 6fbffc0

Browse files
rustyconoverclaude
andcommitted
fixtures: stop paying per-tick and per-reader overhead the tests do not test
- partitioned_sequence, filter_echo_partitioned: declare max_workers as the number of work items. Left unbounded, the client opens one stream per DuckDB thread -- 48 on a 48-core host for at most 24 items, each extra stream an init plus an empty drain, and ~4-5x the worker CPU of threads=1. - split_dynamic_filter: re-parse the init filters per tick under replaying_accepted_state. The framework validated them at init; re-binding every predicate through the evaluator on every tick cost ~10 ms a batch and most of splits/dynamic_filters.test. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent a6e1a5e commit 6fbffc0

3 files changed

Lines changed: 18 additions & 7 deletions

File tree

‎vgi/_test_fixtures/table/filters.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1003,7 +1003,10 @@ def on_init(
10031003
end_idx = min(start_idx + chunk, params.args.count)
10041004
work_items.append(struct.pack(">QQ", start_idx, end_idx))
10051005
params.storage.queue_push(work_items)
1006-
return GlobalInitResponse()
1006+
# No more readers than work items. Left at the default (unbounded) the
1007+
# client opens one stream per DuckDB thread -- 48 on a 48-core host for at
1008+
# most MAX_PARTITIONS items, every extra one an init plus an empty drain.
1009+
return GlobalInitResponse(max_workers=max(1, len(work_items)))
10071010

10081011
@classmethod
10091012
def initial_state(cls, params: ProcessParams[_FilterEchoPartitionedArgs]) -> _FilterEchoPartitionedState:

‎vgi/_test_fixtures/table/sequence.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,10 @@ def on_init(
516516

517517
# Always enqueue (even if empty) to register the invocation
518518
params.storage.queue_push(work_items)
519-
return GlobalInitResponse()
519+
# No more readers than work items. Left at the default (unbounded) the
520+
# client opens one stream per DuckDB thread -- 48 on a 48-core host for at
521+
# most MAX_PARTITIONS items, every extra one an init plus an empty drain.
522+
return GlobalInitResponse(max_workers=max(1, len(work_items)))
520523

521524
@classmethod
522525
def initial_state(cls, params: ProcessParams[PartitionedSequenceFunctionArguments]) -> PartitionedSequenceState:

‎vgi/_test_fixtures/table/splits.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535

3636
from vgi.arguments import Arg
3737
from vgi.cache_control import CacheControl
38+
from vgi.filter_v2 import replaying_accepted_state
3839
from vgi.metadata import FunctionExample, PartitionKind
3940
from vgi.protocol import PlanResponse, ScanSplit, VgiOutputCollector
4041
from vgi.schema_utils import partition_field, schema
@@ -1055,11 +1056,15 @@ def process(
10551056
init = params.init_call
10561057
merged = None
10571058
if init is not None and init.pushdown_filters is not None:
1058-
merged = cls.pushdown_filters(
1059-
init.pushdown_filters,
1060-
join_keys=init.join_keys,
1061-
output_schema=init.output_schema,
1062-
)
1059+
# The framework validated these at init; re-binding every
1060+
# predicate through the evaluator on every tick is what made
1061+
# this fixture cost ~10 ms a batch.
1062+
with replaying_accepted_state():
1063+
merged = cls.pushdown_filters(
1064+
init.pushdown_filters,
1065+
join_keys=init.join_keys,
1066+
output_schema=init.output_schema,
1067+
)
10631068
if merged is None:
10641069
merged = params.current_pushdown_filters
10651070
filter_str = _render_filters_canonical(merged)

0 commit comments

Comments
 (0)