Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions .agents/skills/helm-dev-environment/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,16 @@ above) instead of relying only on `OPENSHELL_GATEWAY_ENDPOINT`; some tests call
### Kubernetes E2E Notes

Use `mise run e2e:kubernetes` for the standard Helm-backed Kubernetes suite.
For the opt-in application listener, build/import matching branch images and run
`mise run e2e:kubernetes:app-listener`; see `e2e/kubernetes/app-listener/README.md`
for image, topology, delivery-mode and takeover inputs. The task owns an isolated
Helm release/namespace, temporary TLS material, in-cluster clients, teardown and
evidence under `results/app-listener/`. It checks policy denial, dedicated Service
selection, shared-owner takeover, gateway-outage expiry and disabled-listener
gateway compatibility, without discovery or a proxy. CI runs four cases under
`test:e2e` and includes them in `Core E2E result`. Its reusable kind setup installs
pinned Calico only for this lane; other jobs retain kindnet. Local k3s already
enforces policy. The lower-level `verify.py` can target a predeployed release.
The kube e2e wrapper creates only one port-forward, to `svc/openshell`; it no
longer forwards the unauthenticated health listener or runs a `/readyz` e2e
target. `/readyz` remains covered by server unit/integration tests.
Expand Down Expand Up @@ -611,3 +621,14 @@ for dependencies still declared in `Chart.yaml`.
| `deploy/kube/manifests/envoy-gateway-openshell.yaml` | GatewayClass and BackendTrafficPolicy for Envoy Gateway (`mise run helm:gateway:apply`) |
| `tasks/scripts/helm-k3s-local.sh` | k3d cluster create/delete/start/stop/status |
| `tasks/scripts/keycloak-k8s-setup.sh` | Keycloak deploy, realm import, and development TLS trust anchor |

## App listener fixture

The opt-in `deploy/helm/openshell/ci/values-app-listener.yaml` enables the
supervisor listener at port 9443 without discovery or a proxy. Provision the
operator TLS Secret and an ingress-client NetworkPolicy first. Follow
`e2e/kubernetes/app-listener/README.md` for direct in-cluster verification,
shared-member selection, and the HA follow-up checks. Keep the fixture disabled
in default profiles. Shared supervisors expose app ports on their Pod IPs
without a shared app Service. Select the member owning the sandbox; automatic
owner selection belongs to the later discovery API.
39 changes: 38 additions & 1 deletion .github/actions/setup-e2e-kind/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ inputs:
description: mise release to install
required: false
default: v2026.9.9
enforce-network-policy:
description: Replace kindnet with pinned Calico for NetworkPolicy assertions
required: false
default: "false"

runs:
using: composite
Expand All @@ -45,11 +49,44 @@ runs:
REGISTRY_PASSWORD: ${{ inputs.registry-password }}
run: echo "$REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$REGISTRY_USERNAME" --password-stdin

- name: Configure policy-enforcing kind network
id: kind-config
if: inputs.enforce-network-policy == 'true'
shell: bash
run: |
set -euo pipefail
config="$RUNNER_TEMP/kind-network-policy.yaml"
cat > "$config" <<'YAML'
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
disableDefaultCNI: true
podSubnet: 10.244.0.0/16
YAML
echo "path=$config" >> "$GITHUB_OUTPUT"

- name: Create kind cluster
uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
with:
cluster_name: ${{ inputs.cluster-name }}
wait: 120s
# Nodes cannot become Ready until the following CNI installation.
wait: ${{ inputs.enforce-network-policy == 'true' && '0s' || '120s' }}
config: ${{ steps.kind-config.outputs.path }}

- name: Install policy-enforcing CNI
if: inputs.enforce-network-policy == 'true'
shell: bash
env:
CLUSTER_NAME: ${{ inputs.cluster-name }}
run: |
set -euo pipefail
curl -fsSL https://raw.githubusercontent.com/projectcalico/calico/v3.30.3/manifests/calico.yaml \
-o "$RUNNER_TEMP/calico.yaml"
sed 's@192.168.0.0/16@10.244.0.0/16@g' "$RUNNER_TEMP/calico.yaml" > "$RUNNER_TEMP/calico-kind.yaml"
kubectl --context "kind-$CLUSTER_NAME" apply -f "$RUNNER_TEMP/calico-kind.yaml"
kubectl --context "kind-$CLUSTER_NAME" -n kube-system rollout status daemonset/calico-node --timeout=300s
kubectl --context "kind-$CLUSTER_NAME" -n kube-system rollout status deployment/calico-kube-controllers --timeout=300s
kubectl --context "kind-$CLUSTER_NAME" wait nodes --all --for=condition=Ready --timeout=120s

- name: Export kind kubeconfig to mise path
shell: bash
Expand Down
38 changes: 38 additions & 0 deletions .github/workflows/branch-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,43 @@ jobs:
agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
test-name: ${{ matrix.test_name }}

kubernetes-app-listener-e2e:
needs: [pr_metadata, build-binaries, build-images]
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
strategy:
fail-fast: false
matrix:
include:
- topology: dedicated
delivery: push
takeover: graceful
name: dedicated push
- topology: dedicated
delivery: poll
takeover: graceful
name: dedicated poll
- topology: shared
delivery: push
takeover: graceful
name: shared graceful takeover
- topology: shared
delivery: push
takeover: abrupt
name: shared abrupt takeover
permissions:
actions: read
contents: read
packages: read
uses: ./.github/workflows/e2e-kubernetes-test.yml
with:
image-tag: ${{ github.sha }}
job-name: Kubernetes app listener (${{ matrix.name }})
e2e-task: e2e:kubernetes:app-listener
enforce-network-policy: true
listener-topology: ${{ matrix.topology }}
listener-delivery: ${{ matrix.delivery }}
listener-takeover: ${{ matrix.takeover }}

kubernetes-workspace-managed-e2e:
needs: [pr_metadata, build-binaries, build-images]
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
Expand Down Expand Up @@ -552,6 +589,7 @@ jobs:
- podman-external-driver-e2e
- vm-external-driver-e2e
- kubernetes-e2e
- kubernetes-app-listener-e2e
- kubernetes-external-driver-e2e
- kubernetes-workspace-managed-e2e
- kubernetes-workspace-operator-e2e
Expand Down
36 changes: 36 additions & 0 deletions .github/workflows/e2e-kubernetes-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,23 @@ on:
required: false
type: string
default: "gateway sandbox supervisor"
enforce-network-policy:
description: "Install a policy-enforcing CNI for listener isolation checks"
required: false
type: boolean
default: false
listener-topology:
required: false
type: string
default: "dedicated"
listener-delivery:
required: false
type: string
default: "push"
listener-takeover:
required: false
type: string
default: "graceful"

permissions:
actions: read
Expand Down Expand Up @@ -129,6 +146,14 @@ jobs:
registry-username: ${{ github.actor }}
registry-password: ${{ secrets.GITHUB_TOKEN }}
mise-version: ${{ inputs.mise-version }}
enforce-network-policy: ${{ inputs.enforce-network-policy }}

- name: Preload listener workload and client fixture
if: inputs.e2e-task == 'e2e:kubernetes:app-listener'
run: |
docker pull --platform linux/amd64 python:3.12-slim
docker image save --platform linux/amd64 --output "$RUNNER_TEMP/listener-python.tar" python:3.12-slim
kind load image-archive "$RUNNER_TEMP/listener-python.tar" --name "$KIND_CLUSTER_NAME"

- name: Run Kubernetes E2E
env:
Expand All @@ -142,4 +167,15 @@ jobs:
IMAGE_TAG: ${{ inputs.image-tag }}
OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell
E2E_TASK: ${{ inputs.e2e-task }}
OPENSHELL_E2E_APP_TOPOLOGY: ${{ inputs.listener-topology }}
OPENSHELL_E2E_APP_DELIVERY: ${{ inputs.listener-delivery }}
OPENSHELL_E2E_APP_TAKEOVER: ${{ inputs.listener-takeover }}
run: mise run --no-deps --skip-deps "$E2E_TASK"

- name: Upload listener test evidence
if: always() && inputs.e2e-task == 'e2e:kubernetes:app-listener'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: listener-${{ inputs.listener-topology }}-${{ inputs.listener-delivery }}-${{ inputs.listener-takeover }}
path: results/app-listener/
if-no-files-found: warn
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

71 changes: 71 additions & 0 deletions crates/openshell-core/src/app_listener.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Driver-independent application listener configuration.

use serde::{Deserialize, Serialize};

/// Explicitly enabled application ingress. Deployment details such as TLS
/// Secret names belong to the driver, not to this common configuration.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct AppListenerConfig {
pub enabled: bool,
pub port: u16,
pub domain: String,
}

impl Default for AppListenerConfig {
fn default() -> Self {
Self {
enabled: false,
port: 8443,
domain: String::new(),
}
}
}

impl AppListenerConfig {
pub fn validate(&self) -> Result<(), String> {
if !self.enabled {
return Ok(());
}
if self.port < 1024 {
return Err("supervisor.app_listener.port must be at least 1024".into());
}
if self.domain.len() > 253
|| !self.domain.split('.').all(|label| {
!label.is_empty()
&& label.len() <= 63
&& !label.starts_with('-')
&& !label.ends_with('-')
&& label
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
})
{
return Err("supervisor.app_listener.domain must be a lowercase DNS name".into());
}
Ok(())
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn only_explicit_enablement_requires_a_valid_endpoint() {
assert!(!AppListenerConfig::default().enabled);
assert!(AppListenerConfig::default().validate().is_ok());
let mut config = AppListenerConfig {
enabled: true,
..Default::default()
};
assert!(config.validate().is_err());
config.domain = "services.example.com".into();
assert!(config.validate().is_ok());
config.port = 0;
assert!(config.validate().is_err());
}
}
2 changes: 2 additions & 0 deletions crates/openshell-core/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,7 @@ pub struct Config {

/// Browser-facing sandbox service routing configuration.
pub service_routing: ServiceRoutingConfig,
pub supervisor_app_listener: crate::app_listener::AppListenerConfig,
}

/// Browser-facing sandbox service routing configuration.
Expand Down Expand Up @@ -1015,6 +1016,7 @@ impl Config {
grpc_rate_limit_requests: None,
grpc_rate_limit_window_secs: None,
service_routing: ServiceRoutingConfig::default(),
supervisor_app_listener: crate::app_listener::AppListenerConfig::default(),
}
}

Expand Down
3 changes: 3 additions & 0 deletions crates/openshell-core/src/grpc_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1863,6 +1863,7 @@ pub struct CachedOpenShellClient {
/// Settings poll result returned by [`CachedOpenShellClient::poll_settings`].
#[derive(Clone, Debug)]
pub struct SettingsPollResult {
pub service_routes: Vec<crate::proto::SandboxServiceRoute>,
pub configuration_instance_id: String,
pub configuration_admitted: bool,
pub configuration_error: String,
Expand Down Expand Up @@ -1890,6 +1891,7 @@ pub struct SettingsPollResult {

fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> SettingsPollResult {
SettingsPollResult {
service_routes: inner.service_routes,
configuration_instance_id: inner.configuration_instance_id,
configuration_admitted: inner.configuration_admitted,
configuration_error: inner.configuration_error,
Expand Down Expand Up @@ -1919,6 +1921,7 @@ fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> Settin
impl From<crate::proto::SandboxConfigSnapshot> for SettingsPollResult {
fn from(snapshot: crate::proto::SandboxConfigSnapshot) -> Self {
settings_poll_result(crate::proto::GetSandboxConfigResponse {
service_routes: snapshot.service_routes,
policy: snapshot.policy,
version: snapshot.version,
policy_hash: snapshot.policy_hash,
Expand Down
1 change: 1 addition & 0 deletions crates/openshell-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
//! - Build version metadata

pub mod activity;
pub mod app_listener;
pub mod auth;
pub mod config;
pub mod container_paths;
Expand Down
Loading
Loading