Merge remote-tracking branch 'origin/main' into protocol-v2-schema-paths

# Conflicts:
#	docs/how-to/iroh.md
#	mkdocs.yml
#	pyproject.toml
#	tests/test_filter_pushdown.py
#	tests/test_serve.py
#	uv.lock
#	vgi/client/client.py
#	vgi/serve.py

Author: rustyconover

.dockerignore

@@ -0,0 +1,15 @@
+.git
+.venv
+.mypy_cache
+.pytest_cache
+.ruff_cache
+site
+__pycache__
+*.pyc
+.tailnet/vgi-rpc/.git
+.env
+.env.*
+*.key
+*.pem
+*.p12
+*.pfx

.github/workflows/tailnet-integration.yml

@@ -0,0 +1,49 @@
+name: Real Tailnet authentication
+
+on:
+  workflow_dispatch:
+    inputs:
+      tailscale_channel:
+        description: Official Tailscale container channel
+        required: true
+        default: stable
+        type: choice
+        options: [stable, unstable]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: vgi-python-real-tailnet
+  cancel-in-progress: false
+
+jobs:
+  tailnet:
+    name: high-level worker / ${{ inputs.tailscale_channel }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    environment: tailnet-integration
+    env:
+      TAILSCALE_IMAGE: tailscale/tailscale:${{ inputs.tailscale_channel }}
+      TS_OAUTH_CLIENT_ID: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+      TS_OAUTH_SECRET: ${{ secrets.TS_OAUTH_SECRET }}
+      TAILNET_ISSUER: ${{ vars.TAILNET_ISSUER }}
+      TAILNET_EXPECTED_CAPABILITY: ${{ vars.TAILNET_EXPECTED_CAPABILITY }}
+      TAILNET_EXPECTED_CLIENT_TAG: ${{ vars.TAILNET_EXPECTED_CLIENT_TAG }}
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Checkout unreleased vgi-rpc identity layer
+        uses: actions/checkout@v7
+        with:
+          repository: Query-farm/vgi-rpc-python
+          path: .tailnet/vgi-rpc
+          ref: 04f19147625b4f643b38a687151d8233daa8997c
+
+      - name: Check kernel networking support
+        run: |
+          test -c /dev/net/tun
+          docker compose version
+
+      - name: Run high-level Tailnet authentication matrix
+        run: tests/tailnet/run.sh

ci/run-integration.sh

@@ -86,6 +86,10 @@ if [ "$TRANSPORT" = "http" ]; then
   #   * buffer_input/sizes.test — input buffering semantics differ over HTTP
   #     (same known limitation). (scale.test_slow is a .test_slow file and is
   #     never staged below, which only finds *.test.)
+  #   * database_worker/package.test — the packaged fixture is deliberately an
+  #     executable wrapper around VGI_TEST_WORKER. An HTTP URL is not an
+  #     executable artifact; the direct-exec lifecycle is covered by all three
+  #     process/socket lanes.
   # cache/revalidate.test now runs on http too: HTTP conditional revalidation is
   # implemented (C++ /init-request validators + vgi-rpc >=0.24.0 surfacing them to
   # the producer's first process()). It needs a community vgi extension carrying
@@ -96,6 +100,7 @@ if [ "$TRANSPORT" = "http" ]; then
     -not -name 'dynamic_filter.test'
     -not -name 'partitioned_sequence.test'
     -not -path './table_in_out/buffer_input/sizes.test'
+    -not -path './database_worker/package.test'
   )
 fi
 
@@ -120,6 +125,16 @@ mkdir -p "$STAGE/test/sql/integration"
     awk -v http="$AWK_HTTP" -f "$HERE/preprocess-require.awk" "$f" > "$STAGE/test/sql/integration/$f"
   done )
 
+# Some integration cases package an executable from the upstream support tree.
+# The staged suite runs with ``$STAGE`` as its working directory, so preserve
+# the relative path used by those SQL fixtures instead of making it depend on
+# the checkout layout outside the stage.
+if [ -f "$VGI_SRC/test/support/database_worker_fixture.sh" ]; then
+  mkdir -p "$STAGE/test/support"
+  cp "$VGI_SRC/test/support/database_worker_fixture.sh" "$STAGE/test/support/"
+  chmod +x "$STAGE/test/support/database_worker_fixture.sh"
+fi
+
 # Empty VGI_RPC_SHM_SIZE_BYTES must not reach the C++ client (it would try to
 # attach a zero-size segment); only a real value enables the shm side channel.
 [ -n "${VGI_RPC_SHM_SIZE_BYTES:-}" ] || unset VGI_RPC_SHM_SIZE_BYTES

docs/how-to/iroh.md

@@ -66,7 +66,7 @@ route, and configure the worker to trust only the intermediary's exact address.
 
 The same high-level client accepts both endpoint schemes:
 
-```python
+```python test="skip"
 from vgi.client import Client
 
 with Client.from_iroh("iroh://<endpoint-id>") as client:
@@ -78,7 +78,7 @@ with Client.from_iroh("httpi://<endpoint-id>/vgi") as client:
 
 For a private relay:
 
-```python
+```python test="skip"
 client = Client.from_iroh(
     "httpi://<endpoint-id>/vgi",
     relay_urls=["https://relay.example.com"],

docs/how-to/tailnet.md

@@ -0,0 +1,195 @@
+---
+description: "How to connect the Python VGI client to workers through Tailscale over HTTPS, direct TCP, or userspace SOCKS5h."
+---
+
+# Run VGI through a Tailnet
+
+The VGI client uses ordinary HTTPS and TCP endpoints. It does not join a
+Tailnet, manage Tailscale credentials, or introduce a `tailscale://` URL
+scheme. Run Tailscale on the client host or in a sidecar and pass the worker's
+MagicDNS name to the normal client factory.
+
+Connectivity and authentication are separate. A client can reach a worker
+through a Tailnet without the worker using Tailnet identity. To authorize from
+the caller's verified Tailscale user, node, tags, or application capabilities,
+configure a peer-identity provider and an authentication policy on the worker.
+
+## Authenticate an HTTP worker
+
+When Tailscale Serve terminates HTTPS, trust its identity headers only from the
+exact local proxy addresses that can reach the application. `require_peer_identity`
+requires verified evidence but deliberately does not invent a stable principal
+for capability-only or login-scoped Serve requests:
+
+```python test="lint"
+from vgi_rpc.http import tailscale_serve_header_provider
+from vgi_rpc.rpc import require_peer_identity
+
+from my_worker import MyWorker
+from vgi.serve import create_app
+
+tailscale = tailscale_serve_header_provider(
+    issuer="tailnet:example",
+    trusted_proxy_addresses=("127.0.0.1", "::1"),
+)
+
+app = create_app(
+    MyWorker,
+    peer_identity_providers=(tailscale,),
+    peer_authentication_policy=require_peer_identity("tailscale"),
+)
+```
+
+Serve login names have `subject_stability="login"`; tagged clients may be
+capability-only and have no subject at all. Consequently, the built-in
+`peer_identity_primary("tailscale")` policy is intentionally unsuitable for
+Serve. If a deployment elects to authenticate from a Serve login, it must use
+an explicit application policy that accepts that weaker identity and scopes it
+to the configured issuer and trusted proxy boundary.
+
+For direct HTTP over a Tailnet, LocalAPI WhoIs can produce a stable user or
+node subject and therefore can be the primary authenticator:
+
+```python test="lint"
+from vgi_rpc.http import tailscale_localapi_provider
+from vgi_rpc.rpc import peer_identity_primary
+
+from my_worker import MyWorker
+from vgi.serve import create_app
+
+tailscale = tailscale_localapi_provider(issuer="tailnet:example")
+app = create_app(
+    MyWorker,
+    peer_identity_providers=(tailscale,),
+    peer_authentication_policy=peer_identity_primary("tailscale"),
+)
+```
+
+`create_app()` remains a standard WSGI application, so the same configuration
+works behind Waitress, Granian, gunicorn, Envoy, nginx, or a cloud HTTP load
+balancer. The provider determines which proxy evidence is trusted; merely
+placing the worker behind a proxy never makes forwarded identity trustworthy.
+
+Trusted-proxy validation must see the physical socket peer. Do not install
+`ProxyFix`, forwarded-address middleware, or an equivalent rewrite ahead of
+VGI's identity middleware: attacker-controlled `Forwarded` or
+`X-Forwarded-For` values must never become `REMOTE_ADDR` before the exact proxy
+allowlist is checked. Keep the backend unreachable except through those
+allowlisted proxies.
+
+## HTTPS through Tailscale Serve
+
+On a host using Tailscale's normal network interface, connect to a worker
+published with Tailscale Serve exactly like any other HTTPS worker:
+
+```python test="lint"
+from vgi.client import Client
+
+with Client.from_http("https://worker.example-tailnet.ts.net") as client:
+    catalogs = client.catalogs()
+```
+
+Tailscale terminates HTTPS and adds its verified Serve identity and application
+capability headers. The VGI server validates those headers against its trusted
+proxy configuration; the client neither constructs nor trusts identity
+headers. Do not send `Tailscale-*` headers from application code.
+
+For userspace networking, inject an HTTP client configured for Tailscale's
+SOCKS5 server. Install SOCKS support with `pip install 'httpx2[socks]'`:
+
+```python test="lint"
+import httpx2
+
+from vgi.client import Client
+
+url = "https://worker.example-tailnet.ts.net"
+with httpx2.Client(
+    base_url=url,
+    proxy="socks5h://127.0.0.1:1055",
+    trust_env=False,
+    timeout=httpx2.Timeout(60.0, connect=15.0),
+) as http_client, Client.from_http(url, httpx_client=http_client) as client:
+    catalogs = client.catalogs()
+```
+
+Using `socks5h` keeps MagicDNS resolution inside the Tailscale sidecar. A proxy
+failure is an error; this configuration does not fall back to a direct route.
+
+## Raw TCP through a Tailnet
+
+Start a high-level Worker with LocalAPI identity using `Worker.serve_tcp()`:
+
+```python test="lint"
+from vgi_rpc.http import tailscale_localapi_provider
+from vgi_rpc.rpc import peer_identity_primary
+
+from my_worker import MyWorker
+
+tailscale = tailscale_localapi_provider(issuer="tailnet:example")
+MyWorker().serve_tcp(
+    "0.0.0.0",
+    9400,
+    threaded=True,
+    max_connections=64,
+    peer_identity_providers=(tailscale,),
+    peer_authentication_policy=peer_identity_primary("tailscale"),
+)
+```
+
+The connection's verified stable Tailnet subject becomes `CallContext.auth` and
+is available to function parameters annotated with `Auth()`. Identity is
+snapshotted once for the stateful TCP connection.
+
+When `threaded=True`, all connections share the same Worker instance. Protect
+mutable subclass state with appropriate synchronization and never store
+per-caller authentication in shared instance attributes. Leave the default
+serial mode in place if the implementation is not safe for concurrent calls.
+
+Behind a Tailscale Service configured to emit PROXY protocol v2, require the
+preamble and name every immediate proxy address explicitly:
+
+```python test="lint"
+MyWorker().serve_tcp(
+    "127.0.0.1",
+    9400,
+    proxy_protocol="required",
+    trusted_proxy_addresses=("127.0.0.1", "::1"),
+    service_name="svc:vgi-worker",
+    peer_identity_providers=(tailscale,),
+    peer_authentication_policy=peer_identity_primary("tailscale"),
+)
+```
+
+Keep that backend unreachable except through the trusted proxy. The asserted
+source is accepted only after immediate-peer validation, and `service_name`
+selects destination-scoped application capabilities in LocalAPI WhoIs.
+
+Direct TCP works with a MagicDNS name when the client host has Tailscale's
+normal network interface:
+
+```python test="lint"
+from vgi.client import Client
+
+with Client.from_tcp("worker.example-tailnet.ts.net", 9400) as client:
+    catalogs = client.catalogs()
+```
+
+For a userspace Tailscale sidecar, use the explicit SOCKS5h option:
+
+```python test="lint"
+with Client.from_tcp(
+    "worker.example-tailnet.ts.net",
+    9400,
+    proxy="socks5h://127.0.0.1:1055",
+) as client:
+    catalogs = client.catalogs()
+```
+
+The proxy applies to ordinary function calls, additional worker connections,
+and catalog discovery. Target hostnames are passed untouched to SOCKS5h and
+are never resolved locally by VGI.
+
+Raw VGI TCP has no application-layer encryption. Tailscale encrypts the
+Tailnet path, but the worker port should remain reachable only through the
+Tailnet and should not be exposed directly to an untrusted network. Prefer
+HTTPS through Tailscale Serve at public or mixed-trust boundaries.

mkdocs.yml

@@ -125,6 +125,7 @@ nav:
     - Expose a catalog: how-to/catalogs.md
     - Persist state across workers: how-to/state-storage.md
     - Serve over HTTP with auth: how-to/http-auth.md
+    - Connect through a Tailnet: how-to/tailnet.md
     - Serve and connect over Iroh: how-to/iroh.md
     - Integrate with the optimizer: how-to/pushdown-and-statistics.md
     - Function Metadata: metadata.md

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.31.2"
+version = "0.31.3"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [
@@ -40,7 +40,7 @@ dependencies = [
     "pyarrow",
     "typer>=0.9",
     "platformdirs",
-    "vgi-rpc>=0.45.0",
+    "vgi-rpc>=0.45.1",
     "httpx2>=2.9.1",
 ]
 
@@ -52,7 +52,7 @@ dependencies = [
 haybarn = ["haybarn>=1.5.5rc1"]
 duckdb = ["duckdb>=1.5.3"]
 http = ["vgi-rpc[http]"]
-iroh = ["vgi-rpc[iroh]"]
+iroh = ["vgi-rpc[http,iroh]"]
 # transport="launch": vgi_rpc.launcher imports filelock unconditionally at
 # module level, which lives behind vgi-rpc's own `cli` extra (alongside
 # typer, already a core dependency here, and httpx2, ditto) rather than in

tests/conformance/test_directory_parity.py

@@ -42,6 +42,15 @@
         "path, ProcessParams.if_none_match threading, and the revalidatable "
         "fixture's 304 behavior)."
     ),
+    "database_worker": (
+        "Resolving database:// package rows, validating artifact digests and "
+        "entrypoints, installing archives, and managing runtime leases are "
+        "C++-extension responsibilities. The Python SDK supplies workers that "
+        "the extension packages and launches, but its Client neither parses "
+        "database:// locations nor owns the extension's artifact cache. The "
+        "cross-language executable/archive cases remain covered by "
+        "vgi/test/sql/integration/database_worker/."
+    ),
     "global_functions": (
         "Publishing worker-declared functions into DuckDB's system.main is "
         "entirely a C++-extension concern: the registry, prefix application, "

tests/tailnet/Dockerfile

@@ -0,0 +1,13 @@
+# syntax=docker/dockerfile:1
+FROM python:3.13-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+COPY .tailnet/vgi-rpc /workspace/vgi-rpc
+COPY . /workspace/vgi-python
+WORKDIR /workspace/vgi-python
+RUN python -m pip install --no-cache-dir "/workspace/vgi-rpc[http]" "/workspace/vgi-python[http]"
+
+CMD ["python", "-m", "tests.tailnet.probe", "--help"]

tests/tailnet/__init__.py

@@ -0,0 +1 @@
+"""Real-Tailnet integration fixtures for the high-level VGI SDK."""