Skip to content

Commit 05e396b

Browse files
committed
Make launcher the default integration transport
1 parent 023906a commit 05e396b

4 files changed

Lines changed: 44 additions & 19 deletions

File tree

‎.github/workflows/integration.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,16 @@ jobs:
5151
fail-fast: true
5252
matrix:
5353
include:
54-
# stdio: the subprocess transport (whole suite — the primary lane).
55-
# launch: the whole suite over the AF_UNIX launcher transport.
54+
# launch: the primary lane, over the reusable AF_UNIX launcher transport.
55+
# stdio: the subprocess compatibility lane (whole suite).
5656
# shm: launch + the POSIX shared-memory side channel. Rides the
5757
# launcher, not raw subprocess: stdio already covers the
5858
# fork-per-connection path and spends most of its wall-clock
5959
# doing it, so re-paying that cost to test shm is waste.
6060
# http: the whole suite over the stateless HTTP transport.
61+
- { lane: launch }
6162
- { lane: stdio }
6263
- { lane: shm, shm_bytes: '67108864' }
63-
- { lane: launch }
6464
- { lane: http }
6565
name: integration (${{ matrix.lane }})
6666
steps:

‎ci/README.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,11 @@ suite against the Python worker (see the project `CLAUDE.md`).
4545

4646
## Transport lanes
4747

48-
`run-integration.sh` honours `TRANSPORT=stdio|shm|launch|http` (the workflow
49-
runs them as a matrix):
48+
`run-integration.sh` honours `TRANSPORT=stdio|shm|launch|http` and defaults to
49+
`launch` when the variable is omitted (the workflow runs every lane explicitly
50+
as a matrix):
5051

51-
- **`stdio`** — the subprocess transport (the primary lane); the whole suite.
52+
- **`stdio`** — the subprocess compatibility lane; the whole suite.
5253
Also boots the versioned and versioned-tables workers as background HTTP
5354
servers (`VGI_VERSIONED_HTTP_WORKER` / `VGI_VERSIONED_TABLES_HTTP_WORKER`) so
5455
the `attach/versioned_tables_*_http` and `versioning_http` tests run.
@@ -62,7 +63,7 @@ runs them as a matrix):
6263
via the `__transport_options__` handshake and carried in POSIX shm rather than
6364
in the pipe or socket — and engages identically over the launcher (verified:
6465
the same 18 `[shm]` transfers under `VGI_RPC_SHM_DEBUG=1` on both).
65-
- **`launch`** — the AF_UNIX launcher transport; the whole suite, with *every*
66+
- **`launch`** — the default and primary AF_UNIX launcher transport; the whole suite, with *every*
6667
`VGI_*_WORKER` fronted by `launch:` so traffic flows through the C++ launcher
6768
(`ResolveLauncherSocketPath` → AF_UNIX → `UnixSocketWorker`). Mirrors the vgi
6869
Makefile's `test_launcher`: the point is that the launcher path produces

‎ci/run-integration.sh‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88
# Ported from vgi-go's harness. The Python differences:
99
# * The "worker binaries" are console scripts installed into the project venv
1010
# by `uv sync --all-extras` (the vgi-fixtures workspace member); the C++
11-
# client spawns them as subprocesses. We use their absolute .venv/bin paths
12-
# so each spawn skips `uv run`'s resolve overhead.
11+
# client launches them from their absolute .venv/bin paths. The default
12+
# launcher transport keeps one warm process instead of paying Python startup
13+
# for every DuckDB connection.
1314
# * Every per-catalog worker is the same Python program with a different
1415
# entrypoint; the base `WorkerClass.main()` understands stdin/stdout,
1516
# `--http`, and the launcher's `--unix`, so one binary covers every lane.
@@ -27,7 +28,7 @@
2728
# HAYBARN_UNITTEST path to the haybarn-unittest binary
2829
# Optional:
2930
# BIN_DIR dir holding the fixture console scripts (default: $REPO/.venv/bin)
30-
# TRANSPORT stdio | shm | launch | http (default: stdio)
31+
# TRANSPORT stdio | shm | launch | http (default: launch)
3132
# VGI_RPC_SHM_SIZE_BYTES shm side-channel segment size (the shm lane)
3233
# STAGE scratch dir for the preprocessed test tree (default: mktemp)
3334
set -euo pipefail
@@ -39,7 +40,7 @@ HERE="$(cd "$(dirname "$0")" && pwd)"
3940
REPO="$(cd "$HERE/.." && pwd)"
4041
BIN_DIR="${BIN_DIR:-$REPO/.venv/bin}"
4142
STAGE="${STAGE:-$(mktemp -d)}"
42-
TRANSPORT="${TRANSPORT:-stdio}"
43+
TRANSPORT="${TRANSPORT:-launch}"
4344
INTEGRATION="$VGI_SRC/test/sql/integration"
4445
[ -d "$INTEGRATION" ] || { echo "::error::no test/sql/integration under VGI_SRC=$VGI_SRC"; exit 1; }
4546

@@ -232,7 +233,7 @@ export VGI_SIMPLE_WRITABLE_WORKER="${LAUNCH_PREFIX}${SIMPLE_WRITABLE}"
232233

233234
case "$TRANSPORT" in
234235
stdio)
235-
# Subprocess transport (the primary lane) — the only lane that spawns a
236+
# Subprocess transport — the only lane that spawns a
236237
# fresh worker process per DuckDB connection, and so the only one that can
237238
# host the crash / pool-recovery tests below.
238239
export VGI_TEST_WORKER="$WORKER"

‎scripts/run_all_tests.sh‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,16 @@
1919
# --integration-only skip pytest
2020
# --quiet don't print summaries to stdout
2121
# --show print cached summaries WITHOUT re-running
22+
#
23+
# Environment:
24+
# VGI_INTEGRATION_TRANSPORT launch (default) | stdio
2225
set -uo pipefail
2326

2427
# Repo locations (override for non-default checkouts).
2528
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
2629
VGI_PYTHON_DIR="${VGI_PYTHON_DIR:-$REPO_ROOT}"
2730
VGI_CPP_DIR="${VGI_CPP_DIR:-$HOME/Development/vgi}"
31+
VGI_INTEGRATION_TRANSPORT="${VGI_INTEGRATION_TRANSPORT:-launch}"
2832

2933
CACHE=/tmp/vgi-test-cache
3034
mkdir -p "$CACHE"
@@ -46,7 +50,7 @@ for arg in "$@"; do
4650
--integration-only) run_pytest=0 ;;
4751
--quiet) quiet=1 ;;
4852
--show) show_only=1 ;;
49-
-h|--help) sed -n '2,22p' "$0"; exit 0 ;;
53+
-h|--help) sed -n '2,26p' "$0"; exit 0 ;;
5054
*) echo "unknown flag: $arg" >&2; exit 2 ;;
5155
esac
5256
done
@@ -186,11 +190,25 @@ run_pytest_job() {
186190

187191
run_integration_job() {
188192
cd "$VGI_CPP_DIR" || exit 99
189-
# `~[.]` excludes Catch2-hidden tests, i.e. `*.test_slow` files. These
190-
# tend to assert best-effort behavior (e.g. `on_cancel` firing) that
191-
# gets flaky under -j 8 contention; run them manually when needed.
192-
VGI_TEST_WORKER="uv run --project $VGI_PYTHON_DIR vgi-fixture-worker" \
193-
timeout 600 ./build/release/test/unittest -j 8 "test/sql/integration/*" "~[.]" >"$INT_LOG" 2>&1
193+
local worker="uv run --project $VGI_PYTHON_DIR vgi-fixture-worker"
194+
case "$VGI_INTEGRATION_TRANSPORT" in
195+
launch)
196+
worker="launch:$worker"
197+
export VGI_REQUIRE_LAUNCHER_TRANSPORT=1
198+
;;
199+
stdio)
200+
unset VGI_REQUIRE_LAUNCHER_TRANSPORT
201+
;;
202+
*)
203+
echo "unsupported VGI_INTEGRATION_TRANSPORT=$VGI_INTEGRATION_TRANSPORT (expected launch or stdio)" >&2
204+
return 2
205+
;;
206+
esac
207+
# `~[.]` excludes Catch2-hidden tests, i.e. `*.test_slow` files. The
208+
# launcher keeps one Python worker warm while unittest advances serially,
209+
# avoiding both repeated startup and cross-test state contention.
210+
VGI_TEST_WORKER="$worker" \
211+
timeout 600 ./build/release/test/unittest "test/sql/integration/*" "~[.]" >"$INT_LOG" 2>&1
194212
local rc=$?
195213
summarize_integration "$INT_LOG" "$INT_SUMMARY" "$INT_FAILURES"
196214
echo "exit=$rc" >>"$INT_SUMMARY"
@@ -206,10 +224,15 @@ if [[ "$run_integration" == 1 ]]; then
206224
pids+=($!)
207225
fi
208226

227+
exit_code=0
209228
for pid in "${pids[@]}"; do
210-
wait "$pid"
229+
if ! wait "$pid"; then
230+
exit_code=1
231+
fi
211232
done
212233

213234
if [[ "$quiet" == 0 ]]; then
214235
print_summary
215236
fi
237+
238+
exit "$exit_code"

0 commit comments

Comments
 (0)