Skip to content

Commit 847dce3

Browse files
rustyconoverclaude
andcommitted
Client: open fan-out connections when a batch is dealt to them
scalar_function and table_in_out_function opened every connection the init's max_workers allowed before dealing a single batch. Scalar and table-in-out functions default to an unbounded max_workers, so the cap was os.cpu_count(): on a 48-core box, a one-batch call opened 48 connections, 47 of them idle. On the subprocess transport that is 47 Python worker processes per call. Through the launcher it is 47 inits queued on one shared worker. _distribute_and_collect now opens a secondary connection the first time a batch is dealt to it and sends its init on that connection's own thread. The round-robin assignment is unchanged (batch i goes to connection i % max_workers), so n batches use min(n, max_workers) connections. table_function has no input to deal and still opens every allowed connection at init. The edge-case tests that allow four workers for fewer batches now also assert how many secondary connections were opened. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 6fbffc0 commit 847dce3

5 files changed

Lines changed: 218 additions & 114 deletions

File tree

‎tests/conftest.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,24 @@ def empty_batch_from_schema(schema: pa.Schema) -> pa.RecordBatch:
132132
return pa.RecordBatch.from_pydict({field.name: [] for field in schema}, schema=schema)
133133

134134

135+
def record_opened_connections(client: Any) -> list[int]:
136+
"""Record the index of each worker connection a started ``client`` opens from now on.
137+
138+
The primary connection (index 0) is opened when the client starts, so call
139+
this afterwards: the list then holds only secondary connections, which a
140+
call opens for fan-out. Only records; each connection is opened for real.
141+
"""
142+
opened: list[int] = []
143+
spawn = client._spawn_worker
144+
145+
def recording_spawn(worker_index: int) -> Any:
146+
opened.append(worker_index)
147+
return spawn(worker_index)
148+
149+
client._spawn_worker = recording_spawn
150+
return opened
151+
152+
135153
# =============================================================================
136154
# Fixtures
137155
# =============================================================================

‎tests/scalar/test_client.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import pyarrow as pa
1010
import pytest
1111

12-
from tests.conftest import assert_total_rows
12+
from tests.conftest import assert_total_rows, record_opened_connections
1313
from vgi import schema
1414
from vgi.arguments import Arguments
1515
from vgi.client import Client
@@ -388,6 +388,7 @@ def test_parallel_double(self, fixture_worker: str) -> None:
388388
batches = [pa.RecordBatch.from_pydict({"x": list(range(i * 100, (i + 1) * 100))}, schema=s) for i in range(10)]
389389

390390
with Client(fixture_worker, worker_limit=4) as client:
391+
opened = record_opened_connections(client)
391392
outputs = list(
392393
client.scalar_function(
393394
function_name="double",
@@ -397,6 +398,9 @@ def test_parallel_double(self, fixture_worker: str) -> None:
397398
)
398399
)
399400

401+
# Ten batches dealt round-robin reach all four connections.
402+
assert opened == [1, 2, 3]
403+
400404
# Should get all 1000 rows back
401405
assert_total_rows(outputs, 1000)
402406

@@ -675,9 +679,12 @@ def test_random_int_multiple_batches(self, fixture_worker: str) -> None:
675679
class TestScalarMultiWorkerEdgeCases:
676680
"""Tests for edge cases with multiple workers for scalar functions.
677681
678-
These tests expose timeout/hang bugs when:
682+
These tests exposed timeout/hang bugs when:
679683
- Processing parquet with one batch of zero rows
680-
- Additional workers spawned but don't receive batches
684+
- Additional workers were spawned but received no batches
685+
686+
The client now opens a secondary connection only when a batch is dealt to
687+
it, so no connection sits idle. The tests check that too.
681688
"""
682689

683690
def test_zero_row_batch_single_worker(self, fixture_worker: str) -> None:
@@ -704,8 +711,9 @@ def test_zero_row_batch_forced_multiple_workers(self, fixture_worker: str) -> No
704711
s = schema(x=pa.int64())
705712
zero_row_batch = pa.RecordBatch.from_pydict({"x": []}, schema=s)
706713

707-
# Force 4 workers even though there's only one batch with zero rows
714+
# Allow 4 workers even though there's only one batch with zero rows
708715
with Client(fixture_worker, worker_limit=4) as client:
716+
opened = record_opened_connections(client)
709717
outputs = list(
710718
client.scalar_function(
711719
function_name="double",
@@ -715,17 +723,19 @@ def test_zero_row_batch_forced_multiple_workers(self, fixture_worker: str) -> No
715723
)
716724
)
717725

718-
# Should complete without hanging
726+
# Should complete without hanging, on the primary alone
719727
assert len(outputs) == 1
720728
assert outputs[0].num_rows == 0
729+
assert opened == []
721730

722731
def test_single_batch_multiple_workers(self, fixture_worker: str) -> None:
723732
"""Single normal batch with max_workers=4 should complete without hanging."""
724733
s = schema(x=pa.int64())
725734
single_batch = pa.RecordBatch.from_pydict({"x": [1, 2, 3]}, schema=s)
726735

727-
# Force 4 workers even though there's only 1 batch
736+
# Allow 4 workers even though there's only 1 batch
728737
with Client(fixture_worker, worker_limit=4) as client:
738+
opened = record_opened_connections(client)
729739
outputs = list(
730740
client.scalar_function(
731741
function_name="double",
@@ -737,15 +747,17 @@ def test_single_batch_multiple_workers(self, fixture_worker: str) -> None:
737747

738748
# Should complete without hanging and return correct data
739749
assert_total_rows(outputs, 3)
750+
assert opened == []
740751

741752
def test_fewer_batches_than_workers(self, fixture_worker: str) -> None:
742753
"""2 batches with max_workers=4 should complete without hanging."""
743754
s = schema(x=pa.int64())
744755
batch1 = pa.RecordBatch.from_pydict({"x": [1, 2]}, schema=s)
745756
batch2 = pa.RecordBatch.from_pydict({"x": [3, 4, 5]}, schema=s)
746757

747-
# Force 4 workers even though there are only 2 batches
758+
# Allow 4 workers even though there are only 2 batches
748759
with Client(fixture_worker, worker_limit=4) as client:
760+
opened = record_opened_connections(client)
749761
outputs = list(
750762
client.scalar_function(
751763
function_name="double",
@@ -757,3 +769,5 @@ def test_fewer_batches_than_workers(self, fixture_worker: str) -> None:
757769

758770
# Should complete without hanging and return correct data (5 rows total)
759771
assert_total_rows(outputs, 5)
772+
# The second batch opened the one secondary connection it needed.
773+
assert opened == [1]

‎tests/table_in_out/test_client.py‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,13 @@
1010
import pyarrow as pa
1111
import pytest
1212

13-
from tests.conftest import SUBPROCESS_FIXTURE_WORKER, assert_single_result, assert_total_rows, make_schema
13+
from tests.conftest import (
14+
SUBPROCESS_FIXTURE_WORKER,
15+
assert_single_result,
16+
assert_total_rows,
17+
make_schema,
18+
record_opened_connections,
19+
)
1420
from vgi.client import Client
1521
from vgi.client.client import ClientError
1622

@@ -146,9 +152,12 @@ def test_large_batch_count(self, fixture_worker: str) -> None:
146152
class TestMultiWorkerEdgeCases:
147153
"""Tests for edge cases with multiple workers.
148154
149-
These tests expose timeout/hang bugs when:
155+
These tests exposed timeout/hang bugs when:
150156
- Processing parquet with one batch of zero rows
151-
- Additional workers spawned but don't receive batches
157+
- Additional workers were spawned but received no batches
158+
159+
The client now opens a secondary connection only when a batch is dealt to
160+
it, so no connection sits idle. The tests check that too.
152161
"""
153162

154163
def test_zero_row_batch_single_worker(self, fixture_worker: str) -> None:
@@ -174,8 +183,9 @@ def test_zero_row_batch_forced_multiple_workers(self, fixture_worker: str) -> No
174183
schema = make_schema([pa.field("id", pa.int64()), pa.field("value", pa.int64())])
175184
zero_row_batch = pa.RecordBatch.from_pydict({"id": [], "value": []}, schema=schema)
176185

177-
# Force 4 workers even though there's only one batch with zero rows
186+
# Allow 4 workers even though there's only one batch with zero rows
178187
with Client(fixture_worker, worker_limit=4) as client:
188+
opened = record_opened_connections(client)
179189
output_batches = list(
180190
client.table_in_out_function(
181191
function_name="echo",
@@ -184,17 +194,19 @@ def test_zero_row_batch_forced_multiple_workers(self, fixture_worker: str) -> No
184194
)
185195
)
186196

187-
# Should complete without hanging
197+
# Should complete without hanging, on the primary alone
188198
assert len(output_batches) >= 1
189199
assert_total_rows(output_batches, 0)
200+
assert opened == []
190201

191202
def test_single_batch_multiple_workers(self, fixture_worker: str) -> None:
192203
"""Single normal batch with max_workers=4 should complete without hanging."""
193204
schema = make_schema([pa.field("id", pa.int64()), pa.field("value", pa.int64())])
194205
single_batch = pa.RecordBatch.from_pydict({"id": [1, 2, 3], "value": [10, 20, 30]}, schema=schema)
195206

196-
# Force 4 workers even though there's only 1 batch
207+
# Allow 4 workers even though there's only 1 batch
197208
with Client(fixture_worker, worker_limit=4) as client:
209+
opened = record_opened_connections(client)
198210
output_batches = list(
199211
client.table_in_out_function(
200212
function_name="echo",
@@ -205,15 +217,17 @@ def test_single_batch_multiple_workers(self, fixture_worker: str) -> None:
205217

206218
# Should complete without hanging and return correct data
207219
assert_total_rows(output_batches, 3)
220+
assert opened == []
208221

209222
def test_fewer_batches_than_workers(self, fixture_worker: str) -> None:
210223
"""2 batches with max_workers=4 should complete without hanging."""
211224
schema = make_schema([pa.field("id", pa.int64()), pa.field("value", pa.int64())])
212225
batch1 = pa.RecordBatch.from_pydict({"id": [1, 2], "value": [10, 20]}, schema=schema)
213226
batch2 = pa.RecordBatch.from_pydict({"id": [3, 4, 5], "value": [30, 40, 50]}, schema=schema)
214227

215-
# Force 4 workers even though there are only 2 batches
228+
# Allow 4 workers even though there are only 2 batches
216229
with Client(fixture_worker, worker_limit=4) as client:
230+
opened = record_opened_connections(client)
217231
output_batches = list(
218232
client.table_in_out_function(
219233
function_name="echo",
@@ -224,6 +238,8 @@ def test_fewer_batches_than_workers(self, fixture_worker: str) -> None:
224238

225239
# Should complete without hanging and return correct data (5 rows total)
226240
assert_total_rows(output_batches, 5)
241+
# The second batch opened the one secondary connection it needed.
242+
assert opened == [1]
227243

228244

229245
class TestWorkerStderrCapture:

0 commit comments

Comments
 (0)