Skip to content

Commit d46a19f

Browse files
committed
test(supervisor): automate Kubernetes application listener e2e
Signed-off-by: Taylor Mutch <taylormutch@gmail.com>
1 parent d94fb6a commit d46a19f

11 files changed

Lines changed: 1570 additions & 3 deletions

File tree

‎.agents/skills/helm-dev-environment/SKILL.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,16 @@ above) instead of relying only on `OPENSHELL_GATEWAY_ENDPOINT`; some tests call
421421
### Kubernetes E2E Notes
422422

423423
Use `mise run e2e:kubernetes` for the standard Helm-backed Kubernetes suite.
424+
For the opt-in application listener, build/import matching branch images and run
425+
`mise run e2e:kubernetes:app-listener`; see `e2e/kubernetes/app-listener/README.md`
426+
for image, topology, delivery-mode and takeover inputs. The task owns an isolated
427+
Helm release/namespace, temporary TLS material, in-cluster clients, teardown and
428+
evidence under `results/app-listener/`. It checks policy denial, dedicated Service
429+
selection, shared-owner takeover, gateway-outage expiry and disabled-listener
430+
gateway compatibility, without discovery or a proxy. CI runs four cases under
431+
`test:e2e` and includes them in `Core E2E result`. Its reusable kind setup installs
432+
pinned Calico only for this lane; other jobs retain kindnet. Local k3s already
433+
enforces policy. The lower-level `verify.py` can target a predeployed release.
424434
The kube e2e wrapper creates only one port-forward, to `svc/openshell`; it no
425435
longer forwards the unauthenticated health listener or runs a `/readyz` e2e
426436
target. `/readyz` remains covered by server unit/integration tests.

‎.github/actions/setup-e2e-kind/action.yml‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ inputs:
2929
description: mise release to install
3030
required: false
3131
default: v2026.9.9
32+
enforce-network-policy:
33+
description: Replace kindnet with pinned Calico for NetworkPolicy assertions
34+
required: false
35+
default: "false"
3236

3337
runs:
3438
using: composite
@@ -45,11 +49,44 @@ runs:
4549
REGISTRY_PASSWORD: ${{ inputs.registry-password }}
4650
run: echo "$REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$REGISTRY_USERNAME" --password-stdin
4751

52+
- name: Configure policy-enforcing kind network
53+
id: kind-config
54+
if: inputs.enforce-network-policy == 'true'
55+
shell: bash
56+
run: |
57+
set -euo pipefail
58+
config="$RUNNER_TEMP/kind-network-policy.yaml"
59+
cat > "$config" <<'YAML'
60+
kind: Cluster
61+
apiVersion: kind.x-k8s.io/v1alpha4
62+
networking:
63+
disableDefaultCNI: true
64+
podSubnet: 10.244.0.0/16
65+
YAML
66+
echo "path=$config" >> "$GITHUB_OUTPUT"
67+
4868
- name: Create kind cluster
4969
uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
5070
with:
5171
cluster_name: ${{ inputs.cluster-name }}
52-
wait: 120s
72+
# Nodes cannot become Ready until the following CNI installation.
73+
wait: ${{ inputs.enforce-network-policy == 'true' && '0s' || '120s' }}
74+
config: ${{ steps.kind-config.outputs.path }}
75+
76+
- name: Install policy-enforcing CNI
77+
if: inputs.enforce-network-policy == 'true'
78+
shell: bash
79+
env:
80+
CLUSTER_NAME: ${{ inputs.cluster-name }}
81+
run: |
82+
set -euo pipefail
83+
curl -fsSL https://raw.githubusercontent.com/projectcalico/calico/v3.30.3/manifests/calico.yaml \
84+
-o "$RUNNER_TEMP/calico.yaml"
85+
sed 's@192.168.0.0/16@10.244.0.0/16@g' "$RUNNER_TEMP/calico.yaml" > "$RUNNER_TEMP/calico-kind.yaml"
86+
kubectl --context "kind-$CLUSTER_NAME" apply -f "$RUNNER_TEMP/calico-kind.yaml"
87+
kubectl --context "kind-$CLUSTER_NAME" -n kube-system rollout status daemonset/calico-node --timeout=300s
88+
kubectl --context "kind-$CLUSTER_NAME" -n kube-system rollout status deployment/calico-kube-controllers --timeout=300s
89+
kubectl --context "kind-$CLUSTER_NAME" wait nodes --all --for=condition=Ready --timeout=120s
5390
5491
- name: Export kind kubeconfig to mise path
5592
shell: bash

‎.github/workflows/branch-e2e.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,43 @@ jobs:
447447
agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
448448
test-name: ${{ matrix.test_name }}
449449

450+
kubernetes-app-listener-e2e:
451+
needs: [pr_metadata, build-binaries, build-images]
452+
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
453+
strategy:
454+
fail-fast: false
455+
matrix:
456+
include:
457+
- topology: dedicated
458+
delivery: push
459+
takeover: graceful
460+
name: dedicated push
461+
- topology: dedicated
462+
delivery: poll
463+
takeover: graceful
464+
name: dedicated poll
465+
- topology: shared
466+
delivery: push
467+
takeover: graceful
468+
name: shared graceful takeover
469+
- topology: shared
470+
delivery: push
471+
takeover: abrupt
472+
name: shared abrupt takeover
473+
permissions:
474+
actions: read
475+
contents: read
476+
packages: read
477+
uses: ./.github/workflows/e2e-kubernetes-test.yml
478+
with:
479+
image-tag: ${{ github.sha }}
480+
job-name: Kubernetes app listener (${{ matrix.name }})
481+
e2e-task: e2e:kubernetes:app-listener
482+
enforce-network-policy: true
483+
listener-topology: ${{ matrix.topology }}
484+
listener-delivery: ${{ matrix.delivery }}
485+
listener-takeover: ${{ matrix.takeover }}
486+
450487
kubernetes-workspace-managed-e2e:
451488
needs: [pr_metadata, build-binaries, build-images]
452489
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
@@ -535,6 +572,7 @@ jobs:
535572
- podman-external-driver-e2e
536573
- vm-external-driver-e2e
537574
- kubernetes-e2e
575+
- kubernetes-app-listener-e2e
538576
- kubernetes-external-driver-e2e
539577
- kubernetes-workspace-managed-e2e
540578
- kubernetes-workspace-operator-e2e

‎.github/workflows/e2e-kubernetes-test.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,23 @@ on:
7777
required: false
7878
type: string
7979
default: "gateway sandbox supervisor"
80+
enforce-network-policy:
81+
description: "Install a policy-enforcing CNI for listener isolation checks"
82+
required: false
83+
type: boolean
84+
default: false
85+
listener-topology:
86+
required: false
87+
type: string
88+
default: "dedicated"
89+
listener-delivery:
90+
required: false
91+
type: string
92+
default: "push"
93+
listener-takeover:
94+
required: false
95+
type: string
96+
default: "graceful"
8097

8198
permissions:
8299
actions: read
@@ -129,6 +146,14 @@ jobs:
129146
registry-username: ${{ github.actor }}
130147
registry-password: ${{ secrets.GITHUB_TOKEN }}
131148
mise-version: ${{ inputs.mise-version }}
149+
enforce-network-policy: ${{ inputs.enforce-network-policy }}
150+
151+
- name: Preload listener workload and client fixture
152+
if: inputs.e2e-task == 'e2e:kubernetes:app-listener'
153+
run: |
154+
docker pull --platform linux/amd64 python:3.12-slim
155+
docker image save --platform linux/amd64 --output "$RUNNER_TEMP/listener-python.tar" python:3.12-slim
156+
kind load image-archive "$RUNNER_TEMP/listener-python.tar" --name "$KIND_CLUSTER_NAME"
132157
133158
- name: Run Kubernetes E2E
134159
env:
@@ -142,4 +167,15 @@ jobs:
142167
IMAGE_TAG: ${{ inputs.image-tag }}
143168
OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell
144169
E2E_TASK: ${{ inputs.e2e-task }}
170+
OPENSHELL_E2E_APP_TOPOLOGY: ${{ inputs.listener-topology }}
171+
OPENSHELL_E2E_APP_DELIVERY: ${{ inputs.listener-delivery }}
172+
OPENSHELL_E2E_APP_TAKEOVER: ${{ inputs.listener-takeover }}
145173
run: mise run --no-deps --skip-deps "$E2E_TASK"
174+
175+
- name: Upload listener test evidence
176+
if: always() && inputs.e2e-task == 'e2e:kubernetes:app-listener'
177+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
178+
with:
179+
name: listener-${{ inputs.listener-topology }}-${{ inputs.listener-delivery }}-${{ inputs.listener-takeover }}
180+
path: results/app-listener/
181+
if-no-files-found: warn

‎e2e/kubernetes/app-listener/README.md‎

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,101 @@ images. This fixture exercises the listener without discovery or the reference
55
proxy. The opt-in Helm values are
66
`deploy/helm/openshell/ci/values-app-listener.yaml`.
77

8+
## Repeatable local checks
9+
10+
The local and CI entry point creates an isolated Helm release, temporary TLS
11+
Secret and client resources, runs the checks, and removes the release/namespace:
12+
13+
```shell
14+
KUBECONFIG="$PWD/kubeconfig" \
15+
IMAGE_TAG=dev OPENSHELL_REGISTRY=openshell \
16+
OPENSHELL_E2E_APP_TOPOLOGY=shared \
17+
OPENSHELL_E2E_APP_TAKEOVER=abrupt \
18+
mise run e2e:kubernetes:app-listener
19+
```
20+
21+
Build and import matching `openshell/gateway:dev`, `openshell/supervisor:dev`,
22+
and `openshell/sandbox:dev` images first, or set the registry/tag to images
23+
already loaded into the cluster. Build the CLI or set `OPENSHELL_BIN` to it.
24+
Set `OPENSHELL_E2E_KUBE_CONTEXT` to select a context explicitly. The cluster must
25+
enforce NetworkPolicy; the local k3s default does. The task installs the pinned
26+
Agent Sandbox controller only when its CRD is absent.
27+
Set `OPENSHELL_E2E_KUBECONFIG` when testing a separate cluster whose kubeconfig
28+
differs from the repository's mise environment.
29+
30+
`OPENSHELL_E2E_APP_TOPOLOGY` accepts `dedicated` (default) or `shared`;
31+
`OPENSHELL_E2E_APP_DELIVERY` accepts `push` (default) or `poll`;
32+
`OPENSHELL_E2E_APP_TAKEOVER` accepts `graceful` (default) or `abrupt` for shared
33+
supervisors. Dedicated runs include gateway-outage expiry. Every run also
34+
disables the listener and verifies absent app sockets/TLS mounts/Services and
35+
continued gateway application ingress. Logs, events, and resource summaries
36+
are written under `results/app-listener/`, without TLS private keys or Secrets.
37+
38+
GitHub Actions runs four matrix cases under the existing `test:e2e` label:
39+
dedicated push/poll and shared graceful/abrupt takeover. These jobs use the
40+
reusable Kubernetes workflow, with kindnet replaced by pinned Calico for policy
41+
enforcement, and participate in `Core E2E result`. CI uploads the evidence even
42+
on failure. They still require the normal current-head `/ok to test <SHA>`
43+
approval mirror.
44+
45+
### Checking an already deployed release
46+
47+
Build the gateway, supervisor, sandbox, and CLI from the branch under test and
48+
deploy them through the local Helm development workflow. Use an isolated release
49+
namespace with gateway TLS, unauthenticated development API access, and the app
50+
listener enabled. Provide the application TLS Secret before installing the
51+
release. The workload and client fixture use `python:3.12-slim`.
52+
53+
Run from the repository root, supplying the public application certificate/CA:
54+
55+
```shell
56+
KUBECONFIG="$PWD/kubeconfig" uv run --no-project python \
57+
e2e/kubernetes/app-listener/verify.py \
58+
--namespace listener-review-shared \
59+
--gateway-endpoint https://localhost:29203 \
60+
--forward-gateway service/listener-review \
61+
--app-ca /path/to/app-ca.crt \
62+
--network-policy \
63+
--takeover graceful
64+
```
65+
66+
Use the actual namespace and gateway Service name of the test release. The
67+
runner maintains its API port-forward across gateway restarts; application
68+
traffic originates in client Pods and uses cluster networking. Application TLS
69+
verifies the supplied CA and hostname. The development API connection uses
70+
`--gateway-insecure` with an isolated temporary CLI configuration.
71+
72+
The runner creates and cleans up its sandboxes, client Pods, public-CA ConfigMap,
73+
and ingress policies. It checks named/unnamed HTTP routes, public Host/Origin and
74+
cookies, credential stripping and explicit bearer passthrough, unknown routes,
75+
Host/SNI mismatch, quiet WebSockets beyond 30 seconds, selective withdrawal, and
76+
re-exposure without restarting the application. When dedicated app Services are
77+
configured, it also verifies their DNS and EndpointSlices.
78+
79+
`--network-policy` requires an enforcing cluster network implementation, such as
80+
the local k3s default. A default kind cluster needs a policy-enforcing CNI before
81+
this assertion can establish isolation.
82+
83+
For a two-member shared pool, `--takeover graceful` deletes the owning Pod and
84+
checks that its sandbox keeps its Pod UID, runtime generation, process, and
85+
in-memory state, while another owner's WebSocket survives. `--takeover abrupt`
86+
first SIGKILLs the supervisor container through the local k3d/kind node's Docker
87+
container, preventing a graceful leave; it requires access to that Docker engine.
88+
These options deliberately interrupt test resources in the supplied namespace.
89+
90+
For dedicated topology, omit `--takeover`. Add
91+
`--expire-gateway statefulset/listener-review` (using the actual workload name)
92+
to scale the isolated gateway to zero, verify route/stream expiry, restore its
93+
original replica count, and check application survival. Repeat with gateway
94+
`config_delivery_mode` set to `push` and `poll`.
95+
96+
The lower-level runner leaves the Helm release and TLS Secret in place. Use
97+
`--disabled --gateway-service <service>` instead of takeover/expiry flags to
98+
check an already disabled release. Configuration rejection and the additional
99+
deployment cleanup scenarios remain in the manual checklist below.
100+
101+
## Manual deployment and additional scenarios
102+
8103
Before deployment, provide an operator-owned `openshell-app-tls` Secret in the
9104
supervisor namespace with `tls.crt` and `tls.key` for
10105
`*.services.example.com`. Keep private keys outside the checkout. Allow a labeled
@@ -63,7 +158,7 @@ that it accepts its own non-controller Sandbox owner reference and has Service
63158
Services must be left untouched, and cleanup uses UID/resource-version
64159
preconditions.
65160

66-
For HA pool validation after integration with #4864, use at least two supervisor
161+
For HA pool validation, use at least two supervisor
67162
members with distinct sandbox owners. Move one owner through graceful downscale
68163
and abrupt loss. Existing streams may disconnect; after selecting the new owner,
69164
new HTTP/WebSocket connections must work without restarting the workload.

‎e2e/kubernetes/app-listener/app.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Loopback HTTP/WebSocket fixture; identity and counter survive supervisor moves."""
5+
6+
import base64
7+
import hashlib
8+
import json
9+
import os
10+
import struct
11+
import uuid
12+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
13+
14+
BOOT = str(uuid.uuid4())
15+
16+
17+
class Handler(BaseHTTPRequestHandler):
18+
protocol_version = "HTTP/1.1"
19+
counter = 0
20+
21+
def do_GET(self):
22+
Handler.counter += 1
23+
if self.headers.get("Upgrade", "").lower() != "websocket":
24+
body = json.dumps(
25+
{
26+
"headers": dict(self.headers),
27+
"boot": BOOT,
28+
"pid": os.getpid(),
29+
"counter": Handler.counter,
30+
}
31+
).encode()
32+
self.send_response(200)
33+
self.send_header("Content-Length", str(len(body)))
34+
self.end_headers()
35+
self.wfile.write(body)
36+
return
37+
# SHA-1 is mandated by the WebSocket handshake, not used for authentication.
38+
accept = base64.b64encode(
39+
hashlib.sha1(
40+
(
41+
self.headers["Sec-WebSocket-Key"]
42+
+ "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
43+
).encode(),
44+
usedforsecurity=False,
45+
).digest()
46+
).decode()
47+
self.send_response(101)
48+
self.send_header("Upgrade", "websocket")
49+
self.send_header("Connection", "Upgrade")
50+
self.send_header("Sec-WebSocket-Accept", accept)
51+
self.end_headers()
52+
while True:
53+
head = self.rfile.read(2)
54+
if len(head) != 2 or head[0] & 15 == 8:
55+
return
56+
length = head[1] & 127
57+
if length == 126:
58+
length = struct.unpack("!H", self.rfile.read(2))[0]
59+
if length > 125 or not head[1] & 128:
60+
return # Fixture deliberately accepts only small masked frames.
61+
mask = self.rfile.read(4)
62+
data = self.rfile.read(length)
63+
if len(mask) != 4 or len(data) != length:
64+
return
65+
data = bytes(value ^ mask[i % 4] for i, value in enumerate(data))
66+
self.wfile.write(bytes([0x81, len(data)]) + data)
67+
self.wfile.flush()
68+
69+
70+
ThreadingHTTPServer(("127.0.0.1", 8080), Handler).serve_forever()

0 commit comments

Comments
 (0)