From 4cd1351e188d9b5d1d8030490a6ca9475ba98c92 Mon Sep 17 00:00:00 2001 From: krishicks Date: Thu, 24 Sep 2026 21:17:12 +0000 Subject: [PATCH 1/7] fix(test): use portable file mode checks in snap installer tests (#3670) The snap installer tests added in #3656 checked the gateway config mode with GNU stat -c, which BSD stat on macOS rejects, so mise run ci failed locally on macOS. Check the mode with find -perm instead, which matches the exact mode on both GNU and BSD systems. Signed-off-by: Kris Hicks --- tasks/scripts/test-install-sh.sh | 2 +- tasks/scripts/test-snap-install-hook.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/scripts/test-install-sh.sh b/tasks/scripts/test-install-sh.sh index f6ab4d2990..9508f1201e 100755 --- a/tasks/scripts/test-install-sh.sh +++ b/tasks/scripts/test-install-sh.sh @@ -371,7 +371,7 @@ if ! grep -Fq 'allow_unauthenticated_users = true' "$snap_config"; then echo "FAIL: Snap gateway config must permit the plaintext local CLI" >&2 exit 1 fi -if [[ $(stat -c '%a' "$snap_config") != 600 ]]; then +if [[ -z $(find "$snap_config" -perm 600) ]]; then echo "FAIL: Snap gateway config must be mode 0600" >&2 exit 1 fi diff --git a/tasks/scripts/test-snap-install-hook.sh b/tasks/scripts/test-snap-install-hook.sh index b2c9c7eab7..aacea97bbb 100755 --- a/tasks/scripts/test-snap-install-hook.sh +++ b/tasks/scripts/test-snap-install-hook.sh @@ -24,7 +24,7 @@ EOF common="${work}/fresh" SNAP_COMMON="$common" "$hook" cmp -s "$expected" "$common/gateway.toml" -if [[ $(stat -c '%a' "$common/gateway.toml") != 600 ]]; then +if [[ -z $(find "$common/gateway.toml" -perm 600) ]]; then echo "FAIL: install hook config must be mode 0600" >&2 exit 1 fi From 52cb8ecee79ca61404c8f9537c99526d07a99652 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 24 Sep 2026 22:41:45 +0000 Subject: [PATCH 2/7] fix(kubernetes): remove NetworkPolicy acknowledgement (#3677) Signed-off-by: Drew Newberry --- .agents/skills/test-release-canary/SKILL.md | 1 - .github/workflows/release-canary.yml | 1 - README.md | 6 +++-- architecture/compute-runtimes.md | 2 +- crates/openshell-driver-kubernetes/README.md | 6 ++--- .../openshell-driver-kubernetes/src/config.rs | 23 ------------------- .../openshell-driver-kubernetes/src/main.rs | 8 ------- deploy/helm/openshell/README.md | 12 ++++++---- deploy/helm/openshell/README.md.gotmpl | 11 +++++---- deploy/helm/openshell/ci/values-keycloak.yaml | 3 +-- .../openshell/ci/values-openshift-scc.yaml | 3 +-- deploy/helm/openshell/ci/values-skaffold.yaml | 1 - .../openshell/templates/gateway-config.yaml | 1 - .../templates/network-policy-ack.yaml | 5 ---- .../tests/network_policy_ack_test.yaml | 20 ---------------- deploy/helm/openshell/values.yaml | 2 -- deploy/helm/test-split-ownership.sh | 2 -- docs/kubernetes/high-availability.mdx | 2 -- docs/kubernetes/ingress.mdx | 2 -- docs/kubernetes/managing-certificates.mdx | 2 -- docs/kubernetes/openshift.mdx | 12 +++++++--- docs/kubernetes/sandbox-runtime.mdx | 7 +++--- docs/kubernetes/setup.mdx | 20 ++++++++-------- docs/reference/gateway-config.mdx | 8 ++++--- docs/reference/sandbox-compute-drivers.mdx | 6 ++++- .../Dockerfile.external-kubernetes-gateway | 3 +-- examples/gateway-deploy-connect.md | 2 +- skills/debug-openshell-cluster/SKILL.md | 5 ++-- tasks/helm.toml | 4 ++-- 29 files changed, 64 insertions(+), 116 deletions(-) delete mode 100644 deploy/helm/openshell/templates/network-policy-ack.yaml delete mode 100644 deploy/helm/openshell/tests/network_policy_ack_test.yaml diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 2a996bf0b5..4b95b805f8 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -109,7 +109,6 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --namespace openshell --create-namespace \ --set server.disableTls=true \ --set server.telemetryEnabled=false \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --wait --timeout 5m kubectl wait --namespace openshell \ diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 723bb4b005..239f49eced 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -359,7 +359,6 @@ jobs: --set server.disableTls=true \ --set server.auth.allowUnauthenticatedUsers=true \ --set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}" \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --wait --timeout 5m - name: Verify gateway pod is Ready diff --git a/README.md b/README.md index 6049b609a3..4972acad1d 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,14 @@ The installer installs the latest stable release by default. See [Prerelease and **Kubernetes installation:** > **Experimental** — the Kubernetes deployment path is under active development. Expect rough edges and breaking changes. +> **Required:** Your cluster CNI MUST enforce Kubernetes `NetworkPolicy` for +> ingress and egress in every sandbox namespace. OpenShell creates the policies, +> but Kubernetes does not verify that the CNI applies them. Deploy the OpenShell gateway into a Kubernetes cluster from the OCI chart published to GHCR: ```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart ``` See [`deploy/helm/openshell/README.md`](deploy/helm/openshell/README.md) for available versions, dev tag conventions, and configuration. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f4745b3ac7..71a43bce2a 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -301,7 +301,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. |---|---|---|---| | Docker | Local development with Docker available. | Capability-free workload container. | Uses `network_mode=none`; a separate capability-free supervisor container mediates egress and access over a private daemon-local Unix socket volume. | | Podman | Existing rootless driver. | Container. | Not converted by this isolation stack. | -| Kubernetes | Cluster deployment through Helm. | Capability-free sandbox Pod. | Uses one namespace-wide empty-egress workload NetworkPolicy and a separate capability-free supervisor Pod over mutually authenticated TLS. It requires an enforcing CNI and trusted sandbox namespace. | +| Kubernetes | Cluster deployment through Helm. | Capability-free sandbox Pod. | Always creates a namespace-wide empty-egress workload NetworkPolicy and a separate capability-free supervisor Pod over mutually authenticated TLS. It requires an enforcing CNI and trusted sandbox namespace; the Kubernetes API does not attest policy enforcement. | | VM | Experimental microVM isolation. | Per-sandbox libkrun or QEMU VM. | The NIC-less guest runs `openshell-sandbox` as PID 1; host `openshell-supervisor` owns gateway networking and reaches the guest over vsock. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 084f197fb6..a71fa55d0a 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -81,9 +81,9 @@ and permits OpenShell supervisor Pods to reach the sandbox TLS port. The authenticated Sandbox Protocol binds each connection to the exact sandbox and supervisor Pod identities. Supervisors have normal egress for gateway, DNS, and policy-approved upstream connections unless an operator policy restricts -them. Set -`sandbox_runtime.network_policy_enforced = true` only after verifying that the cluster -CNI enforces ingress and egress `NetworkPolicy` for sandbox namespaces. +them. The cluster CNI must enforce ingress and egress `NetworkPolicy` for every +sandbox namespace. Kubernetes accepts policy objects without confirming +enforcement, so operators must verify CNI support before running sandboxes. Each sandbox generation uses two immutable bootstrap Secrets. A trusted init container stages the sandbox bootstrap into memory, and the sandbox removes it diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index c9697021ae..40bcdc8422 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -78,9 +78,6 @@ pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesSandboxRuntimeConfig { - /// Explicit operator assertion that the cluster CNI enforces - /// `networking.k8s.io/v1` `NetworkPolicy` for the sandbox namespaces. - pub network_policy_enforced: bool, /// TCP port exposed by the workload boundary to its paired control pod. pub boundary_port: u16, } @@ -88,7 +85,6 @@ pub struct KubernetesSandboxRuntimeConfig { impl Default for KubernetesSandboxRuntimeConfig { fn default() -> Self { Self { - network_policy_enforced: false, boundary_port: 5500, } } @@ -96,12 +92,6 @@ impl Default for KubernetesSandboxRuntimeConfig { impl KubernetesSandboxRuntimeConfig { pub fn validate(&self) -> Result<(), String> { - if !self.network_policy_enforced { - return Err( - "sandbox_runtime.network_policy_enforced must be true after the operator has verified CNI NetworkPolicy enforcement" - .to_string(), - ); - } if self.boundary_port < 1024 { return Err("sandbox_runtime.boundary_port must be at least 1024".to_string()); } @@ -887,19 +877,6 @@ mod tests { assert!(cfg.workspace_storage_class.is_empty()); } - #[test] - fn sandbox_runtime_requires_network_policy_enforcement_acknowledgement() { - let mut cfg = KubernetesComputeConfig::default(); - assert!( - cfg.validate_proxy_uid() - .unwrap_err() - .contains("network_policy_enforced") - ); - - cfg.sandbox_runtime.network_policy_enforced = true; - cfg.validate_proxy_uid().unwrap(); - } - #[test] fn serde_rejects_sidecar_binary_identity_field() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index fcb8faa1e6..7fbce18fee 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -128,13 +128,6 @@ struct Args { #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] supervisor_image_pull_policy: Option, - #[arg( - long, - env = "OPENSHELL_K8S_SANDBOX_RUNTIME_NETWORK_POLICY_ENFORCED", - default_value_t = false - )] - sandbox_runtime_network_policy_enforced: bool, - #[arg( long, env = "OPENSHELL_K8S_SANDBOX_RUNTIME_BOUNDARY_PORT", @@ -270,7 +263,6 @@ async fn main() -> Result<()> { .unwrap_or_else(openshell_core::config::default_supervisor_image), supervisor_image_pull_policy: args.supervisor_image_pull_policy, sandbox_runtime: KubernetesSandboxRuntimeConfig { - network_policy_enforced: args.sandbox_runtime_network_policy_enforced, boundary_port: args.sandbox_runtime_boundary_port, }, https_proxy: args.https_proxy, diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index f90359832f..4ab3517b27 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -19,6 +19,12 @@ multiple pre-provisioned workspace namespaces. ## Prerequisites +> **Required:** Your cluster CNI MUST enforce Kubernetes `NetworkPolicy` for +> ingress and egress in every sandbox namespace. OpenShell creates the policies, +> but Kubernetes accepts them even if no CNI enforces them. Without enforcement, +> sandbox workloads may connect directly and bypass supervisor network policy. +> Verify CNI support before installing OpenShell. + The Kubernetes Agent Sandbox CRDs and controller must be installed on the cluster before deploying OpenShell. Install them with: ```shell @@ -35,8 +41,7 @@ where Helm cannot discover cluster APIs. ## Install on Kubernetes ```shell -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version ``` ## Install on OpenShift @@ -49,7 +54,6 @@ oc create ns openshell # Deploy openshell with overrides to allow SCC assignment of fsGroup and runAsUser for the gateway helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -110,7 +114,6 @@ Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ -n openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set workload.kind=deployment \ --set server.externalDbSecret=my-pg-credentials ``` @@ -352,7 +355,6 @@ discovery endpoint or its TLS CA. | supervisor.image.repository | string | `"openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | | supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. | -| supervisor.sandboxRuntime.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. | | tolerations | list | `[]` | Tolerations for the gateway pod. | | upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"caBundle":{"configMapName":"","key":"ca.crt"},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | | upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 59e8118f3b..6e49260c20 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -19,6 +19,12 @@ multiple pre-provisioned workspace namespaces. ## Prerequisites +> **Required:** Your cluster CNI MUST enforce Kubernetes `NetworkPolicy` for +> ingress and egress in every sandbox namespace. OpenShell creates the policies, +> but Kubernetes accepts them even if no CNI enforces them. Without enforcement, +> sandbox workloads may connect directly and bypass supervisor network policy. +> Verify CNI support before installing OpenShell. + The Kubernetes Agent Sandbox CRDs and controller must be installed on the cluster before deploying OpenShell. Install them with: ```shell @@ -35,8 +41,7 @@ where Helm cannot discover cluster APIs. ## Install on Kubernetes ```shell -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version ``` ## Install on OpenShift @@ -49,7 +54,6 @@ oc create ns openshell # Deploy openshell with overrides to allow SCC assignment of fsGroup and runAsUser for the gateway helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -110,7 +114,6 @@ Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ -n openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set workload.kind=deployment \ --set server.externalDbSecret=my-pg-credentials ``` diff --git a/deploy/helm/openshell/ci/values-keycloak.yaml b/deploy/helm/openshell/ci/values-keycloak.yaml index ae1810f414..6df74e3258 100644 --- a/deploy/helm/openshell/ci/values-keycloak.yaml +++ b/deploy/helm/openshell/ci/values-keycloak.yaml @@ -8,8 +8,7 @@ # # Then layer this file on top of values.yaml when deploying: # helm upgrade --install openshell . \ -# -f values.yaml -f ci/values-skaffold.yaml -f ci/values-keycloak.yaml \ -# --set supervisor.sandboxRuntime.networkPolicyEnforced=true +# -f values.yaml -f ci/values-skaffold.yaml -f ci/values-keycloak.yaml # # Or add this file to skaffold.yaml valuesFiles for iterative dev. # diff --git a/deploy/helm/openshell/ci/values-openshift-scc.yaml b/deploy/helm/openshell/ci/values-openshift-scc.yaml index 8f1a8d07d6..b7f37be6e0 100644 --- a/deploy/helm/openshell/ci/values-openshift-scc.yaml +++ b/deploy/helm/openshell/ci/values-openshift-scc.yaml @@ -4,8 +4,7 @@ # OpenShift SCC compatibility overlay. Removes the hardcoded runAsUser and # fsGroup so that OpenShift's restricted-v2 SCC can inject the namespace- # assigned UID/GID range. Layer after values.yaml: -# helm install openshell deploy/helm/openshell -f ci/values-openshift-scc.yaml \ -# --set supervisor.sandboxRuntime.networkPolicyEnforced=true +# helm install openshell deploy/helm/openshell -f ci/values-openshift-scc.yaml # # The e2e Kubernetes harness applies this automatically when it detects an # OpenShift cluster (route.openshift.io API present). diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index c3c1a84a4e..8def2796a4 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -20,4 +20,3 @@ supervisor: # The local k3s cluster created by `mise run helm:k3s:create` enables its # built-in NetworkPolicy controller for sandbox namespaces. sandboxRuntime: - networkPolicyEnforced: true diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 13b453041b..b1ec38bd0a 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -232,7 +232,6 @@ data: gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } [openshell.drivers.kubernetes.sandbox_runtime] - network_policy_enforced = {{ .Values.supervisor.sandboxRuntime.networkPolicyEnforced }} boundary_port = {{ .Values.supervisor.sandboxRuntime.boundaryPort | default 5500 }} {{- if not $credentialDrivers }} diff --git a/deploy/helm/openshell/templates/network-policy-ack.yaml b/deploy/helm/openshell/templates/network-policy-ack.yaml deleted file mode 100644 index 780d4314c9..0000000000 --- a/deploy/helm/openshell/templates/network-policy-ack.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -{{- if not .Values.supervisor.sandboxRuntime.networkPolicyEnforced }} -{{- fail "supervisor.sandboxRuntime.networkPolicyEnforced must be true after you verify that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace" }} -{{- end }} diff --git a/deploy/helm/openshell/tests/network_policy_ack_test.yaml b/deploy/helm/openshell/tests/network_policy_ack_test.yaml deleted file mode 100644 index 12ca399e73..0000000000 --- a/deploy/helm/openshell/tests/network_policy_ack_test.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -suite: NetworkPolicy enforcement acknowledgement -templates: - - templates/network-policy-ack.yaml -tests: - - it: rejects an install without operator acknowledgement - set: - supervisor.sandboxRuntime.networkPolicyEnforced: false - asserts: - - failedTemplate: - errorMessage: supervisor.sandboxRuntime.networkPolicyEnforced must be true after you verify that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace - - - it: accepts an acknowledged CNI - set: - supervisor.sandboxRuntime.networkPolicyEnforced: true - asserts: - - hasDocuments: - count: 0 diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 34067eb31d..b48a7c269f 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -70,8 +70,6 @@ supervisor: # -- Supervisor image digest. When set, this takes precedence over tag. digest: "" sandboxRuntime: - # -- Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. - networkPolicyEnforced: false # -- Workload boundary TLS listener port. boundaryPort: 5500 diff --git a/deploy/helm/test-split-ownership.sh b/deploy/helm/test-split-ownership.sh index a7a865363b..30fd7360d4 100755 --- a/deploy/helm/test-split-ownership.sh +++ b/deploy/helm/test-split-ownership.sh @@ -11,7 +11,6 @@ trap 'rm -rf "${work_dir}"' EXIT helm template openshell "${repo_root}/deploy/helm/openshell" \ --namespace openshell \ --set agentSandbox.preflight.enabled=false \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set workspaceResources.enabled=false \ >"${work_dir}/gateway.yaml" @@ -41,7 +40,6 @@ fi helm template openshell "${repo_root}/deploy/helm/openshell" \ --namespace openshell \ --set agentSandbox.preflight.enabled=false \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set-json workspaceResources=null \ >"${work_dir}/legacy-reuse-values.yaml" diff --git a/docs/kubernetes/high-availability.mdx b/docs/kubernetes/high-availability.mdx index c45ce9a1f6..f83b181462 100644 --- a/docs/kubernetes/high-availability.mdx +++ b/docs/kubernetes/high-availability.mdx @@ -90,7 +90,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --values values-ha.yaml \ --wait ``` @@ -196,7 +195,6 @@ helm upgrade openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --values values-ha.yaml \ --wait ``` diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index fd6be78fbd..c9e081065b 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -57,7 +57,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set grpcRoute.enabled=true \ --set grpcRoute.gateway.create=true \ --set grpcRoute.gateway.className=eg @@ -114,7 +113,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set grpcRoute.enabled=true \ --set grpcRoute.gateway.create=true \ --set grpcRoute.gateway.className=eg \ diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index 13815e5cf9..322815cbf1 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -52,7 +52,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set certManager.enabled=true ``` @@ -74,7 +73,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set certManager.enabled=true \ --set certManager.serverIssuerRef.name=letsencrypt-prod \ --set certManager.serverIssuerRef.kind=ClusterIssuer \ diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index e510fc44c4..ecfddefd0e 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -41,17 +41,23 @@ in the sandbox qualification output. - [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs. - A CNI that enforces ingress and egress `NetworkPolicy` in sandbox namespaces. + +Your cluster MUST enforce ingress and egress `NetworkPolicy` in every sandbox +namespace. OpenShell creates the policies, but Kubernetes does not verify that +the CNI applies them. Without enforcement, sandbox workloads may bypass +supervisor network policy through direct connections. + + ## Install OpenShell Pre-create the namespace, then install the chart. Keep the default restricted -security posture and acknowledge NetworkPolicy only after validating the CNI. +security posture. ```shell oc create ns openshell helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ - --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true + --namespace openshell ``` The driver reads the namespace's `openshift.io/sa.scc.uid-range` annotation and diff --git a/docs/kubernetes/sandbox-runtime.mdx b/docs/kubernetes/sandbox-runtime.mdx index 1376916b44..ac22288af4 100644 --- a/docs/kubernetes/sandbox-runtime.mdx +++ b/docs/kubernetes/sandbox-runtime.mdx @@ -81,9 +81,10 @@ restricts them. Kubernetes policies are additive. Keep sandbox namespaces under administrative control so another principal cannot add permissive policies, create Pods with -OpenShell labels, or read bootstrap Secrets. Set -`supervisor.sandboxRuntime.networkPolicyEnforced: true` only after you verify that the -cluster CNI enforces both ingress and egress policies for these namespaces. +OpenShell labels, or read bootstrap Secrets. The cluster CNI MUST enforce both +ingress and egress `NetworkPolicy` in every sandbox namespace. Kubernetes accepts +policy objects even when no CNI enforces them; OpenShell does not test traffic +to verify enforcement. ## Bootstrap a Sandbox diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 95c3e6c620..dc9a138207 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -12,6 +12,14 @@ position: 1 The OpenShell Helm chart is experimental and under active development. Templates, values, and defaults can change between releases. Do not use it in production. + +Your cluster MUST use a CNI that enforces Kubernetes `NetworkPolicy` for both +ingress and egress in every sandbox namespace. OpenShell creates the policies, +but Kubernetes accepts them even when no CNI enforces them. Without enforcement, +sandbox workloads may reach the network directly and bypass supervisor policy. +Verify CNI support before installing OpenShell. + + Use the Kubernetes deployment when the gateway should run on a shared cluster, in a cloud environment, or as part of team infrastructure. The Helm chart handles PKI bootstrap, RBAC, sandbox namespace setup, and the gateway workload. It uses a StatefulSet by default for the SQLite database, and can render a Deployment when `server.externalDbSecret` points at an external database. ## Prerequisites @@ -21,6 +29,7 @@ Make sure the following are in place before you install. | Prerequisite | Required | Notes | |---|---|---| | Kubernetes 1.29+ with RBAC enabled | Yes | No additional notes. | +| CNI that enforces ingress and egress `NetworkPolicy` in sandbox namespaces | Yes | Verify enforcement on your cluster. | | Helm 3.x | Yes | No additional notes. | | Agent Sandbox controller and CRDs | Yes | Install before the OpenShell chart. Refer to [Install Agent Sandbox](#install-agent-sandbox). | | cert-manager | No | Refer to [Managing Certificates](/kubernetes/managing-certificates). Use cert-manager only if you prefer it over the built-in PKI job. | @@ -85,8 +94,7 @@ Install from the OCI registry on GHCR. Replace `` with the chart versio helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ - --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true + --namespace openshell ``` To use the latest development build instead of a stable release: @@ -95,8 +103,7 @@ To use the latest development build instead of a stable release: helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version 0.0.0-dev \ - --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true + --namespace openshell ``` The chart automatically generates PKI secrets on first install using pre-install Helm hooks. No manual secret creation is required. @@ -111,7 +118,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set workspaceResources.enabled=false \ --set server.sandboxNamespace=app-a @@ -149,7 +155,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set server.credentialDrivers.kubernetesSecrets.enabled=true \ --set server.credentialDrivers.kubernetesSecrets.namespace=openshell-credentials \ --set server.credentialDrivers.kubernetesSecrets.createNamespace=true @@ -237,7 +242,6 @@ The most commonly changed values are: | `server.auth.allowUnauthenticatedUsers` | Accept user-facing calls without OIDC or mTLS credentials. Use only for trusted local development or a fully trusted access proxy. | | `server.enableLoopbackServiceHttp` | Enable local plaintext HTTP for loopback sandbox service URLs. Defaults to `true`. | | `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. | -| `supervisor.sandboxRuntime.networkPolicyEnforced` | Required acknowledgement that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. | | `supervisor.sandboxRuntime.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. | | `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | @@ -254,7 +258,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --values my-values.yaml ``` @@ -350,7 +353,6 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set serviceAccount.create=false \ --set serviceAccount.name=my-existing-sa ``` diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 06404a69de..e032ff8a89 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -791,13 +791,15 @@ gateway_namespace = "openshell" gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" } [openshell.drivers.kubernetes.sandbox_runtime] -# Required acknowledgement that the cluster CNI enforces NetworkPolicy and the -# sandbox namespaces prevent untrusted policy, pod, label, and Secret changes. -network_policy_enforced = true # TLS-protected boundary listener reached only by the paired control pod. boundary_port = 5500 ``` +The Kubernetes driver always creates the workload `NetworkPolicy` before +releasing a sandbox Pod. Your cluster CNI must enforce ingress and egress +`NetworkPolicy` in every sandbox namespace; the Kubernetes API does not verify +enforcement. + In managed workspace mode, the Kubernetes driver copies each explicitly named `image_pull_secrets` Secret from `namespace` into the managed workspace namespace on sandbox creation. Shared and operator modes require the Secret to diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 383f24a329..395a04edf8 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -442,6 +442,11 @@ than being staged into the guest. See the [Gateway Configuration File](./gateway Kubernetes-backed sandboxes run as pods in the configured sandbox namespace. Use Kubernetes for shared clusters, remote compute, GPU scheduling, and operator-managed environments. +The cluster CNI MUST enforce ingress and egress `NetworkPolicy` in every sandbox +namespace. OpenShell creates the workload policy, but the Kubernetes API cannot +confirm enforcement. Without it, workload pods may connect directly and bypass +supervisor network policy. + Kubernetes workspace namespaces are an administrative trust boundary. In shared and managed modes, only the OpenShell gateway and its trusted Agent Sandbox controller may administer Sandbox CRs, sandbox pods, or the configured @@ -471,7 +476,6 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `sandbox_runtime_image_pull_policy` | `sandboxRuntime.image.pullPolicy` | Set the Kubernetes image pull policy for the sandbox runtime image. | | `supervisor_image` | `supervisor.image.registry` / `supervisor.image.repository` / `supervisor.image.tag` / `supervisor.image.digest` | Override the image that provides `openshell-supervisor`. Individual image values take precedence over `global.image`; a digest takes precedence over the tag. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | -| `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | | `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` or `https://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | | `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. | diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway index 3b860dacc7..ffa1072bb7 100644 --- a/e2e/docker/Dockerfile.external-kubernetes-gateway +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -24,8 +24,7 @@ ENV OPENSHELL_COMPUTE_DRIVER=kubernetes \ OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_SANDBOX_RUNTIME_IMAGE=${SANDBOX_RUNTIME_IMAGE} \ OPENSHELL_SANDBOX_RUNTIME_IMAGE_PULL_POLICY=if_not_present \ - OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \ - OPENSHELL_K8S_SANDBOX_RUNTIME_NETWORK_POLICY_ENFORCED=true + OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container USER 1000:1000 EXPOSE 8080 diff --git a/examples/gateway-deploy-connect.md b/examples/gateway-deploy-connect.md index 2861444d1c..2ad4f29303 100644 --- a/examples/gateway-deploy-connect.md +++ b/examples/gateway-deploy-connect.md @@ -6,6 +6,7 @@ Deploy or register an OpenShell gateway, verify it is reachable, and run your fi - OpenShell CLI installed (`openshell`) - A reachable gateway endpoint, or access to a Kubernetes cluster where you can install the Helm chart +- For Kubernetes installs, a CNI that enforces ingress and egress `NetworkPolicy` in sandbox namespaces ## Helm Deployment @@ -15,7 +16,6 @@ Install the gateway into a Kubernetes cluster you manage: kubectl create namespace openshell helm upgrade --install openshell deploy/helm/openshell \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set server.disableTls=true \ --set service.type=ClusterIP ``` diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 406e7209d2..22e35fbb27 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -708,9 +708,8 @@ kubectl -n get sandbox -o jsonpath='{.spec.te ``` The Kubernetes driver creates a sandbox workload Pod and a separate, directly -managed supervisor Pod. Helm must render -`network_policy_enforced = true`. This is an explicit operator assertion that -the cluster CNI enforces Kubernetes NetworkPolicy; the Kubernetes API cannot +managed supervisor Pod. The cluster CNI must enforce ingress and egress +Kubernetes NetworkPolicy in every sandbox namespace; the Kubernetes API cannot attest enforcement. Run sandboxes only in a trusted namespace where tenants cannot create Pods, copy OpenShell role labels, or read the bootstrap Secret. diff --git a/tasks/helm.toml b/tasks/helm.toml index 33b3550dff..43f01bf3f1 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -39,12 +39,12 @@ run = """ helm dependency build deploy/helm/openshell echo "--- helm lint: defaults ---" echo "values files: deploy/helm/openshell/values.yaml" - helm lint deploy/helm/openshell --set agentSandbox.preflight.enabled=false --set supervisor.sandboxRuntime.networkPolicyEnforced=true + helm lint deploy/helm/openshell --set agentSandbox.preflight.enabled=false for f in deploy/helm/openshell/ci/values-*.yaml; do variant=$(basename "$f" .yaml | sed 's/values-//') echo "--- helm lint: $variant ---" echo "values files: deploy/helm/openshell/values.yaml, $f" - helm lint deploy/helm/openshell -f "$f" --set agentSandbox.preflight.enabled=false --set supervisor.sandboxRuntime.networkPolicyEnforced=true + helm lint deploy/helm/openshell -f "$f" --set agentSandbox.preflight.enabled=false done echo "--- helm lint: workspace defaults ---" helm lint deploy/helm/openshell-workspace From a00ea31c66de4dbe508b8736a9b05b93ce37a814 Mon Sep 17 00:00:00 2001 From: Jim Meyer Date: Thu, 24 Sep 2026 22:42:51 +0000 Subject: [PATCH 3/7] ci: restrict copy-pr-bot manual vetters (#3678) Signed-off-by: Jim Meyer --- .github/copy-pr-bot.yaml | 18 ++++++++++++++++++ CI.md | 20 +++++++++++++------- architecture/build.md | 4 ++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml index 4cfbdc7f05..a4e9281742 100644 --- a/.github/copy-pr-bot.yaml +++ b/.github/copy-pr-bot.yaml @@ -1,3 +1,21 @@ enabled: true auto_sync_draft: false auto_sync_ready: true +vetters_override: + - alangou + - derekwaynecarr + - drew + - elezar + - johnnygreco + - johntmyers + - kirit93 + - krishicks + - matthewgrossman + - mrunalp + - pimlock + - purp + - SDAChess + - shailendra-nv + - sjenning + - TaylorMutch + - zredlined diff --git a/CI.md b/CI.md index 7555935a9c..72f63340ff 100644 --- a/CI.md +++ b/CI.md @@ -8,6 +8,12 @@ For local test commands see [TESTING.md](TESTING.md). For PR conventions see [CO PR CI that runs on NVIDIA self-hosted runners uses NVIDIA's copy-pr-bot. The bot mirrors trusted PR commits to internal `pull-request/` branches in this repository. The gated workflows trigger on pushes to those branches, not on the original PR. +When a PR is not mirrored automatically, only the GitHub users listed in +`.github/copy-pr-bot.yaml` under `vetters_override` can admit its current +revision with `/ok to test `. The list is a snapshot of the codeowners +and selected repository maintainers; update it when those people change. +This setting does not change the bot's automatic trust policy for ready PRs. + `Branch Checks` run automatically after copy-pr-bot mirrors the PR. `Required CI Gates` posts PR-head statuses that verify the mirror exists, is current, and ran the expected push-based workflows. E2E suites are opt-in because they are more expensive and publish temporary images. Merge queue validation is a second integration gate for `main`. After a PR has passed the required PR-head statuses, a maintainer adds it to the merge queue. GitHub creates a temporary merge-group branch that combines the latest `main`, the queued PR, and any earlier queued PRs. The same required `OpenShell / ...` status contexts are then published against the merge-group SHA before GitHub merges it. @@ -313,21 +319,21 @@ Flow: 6. New commits push to the mirror automatically and re-trigger `Branch Checks` plus any labeled E2E jobs in `Branch E2E Checks`. 7. When the PR is ready to merge, use **Add to merge queue** instead of merging directly. The queue validates the final integration state before updating `main`. -### Forked PR +### PR requiring manual admission Prerequisites: -- DCO sign-off (`git commit -s`) on every commit. Commit signing is not required for forks - copy-pr-bot trusts forks based on maintainer review, not signing. -- A maintainer must vouch you. See the [Vouch System](AGENTS.md#vouch-system). +- DCO sign-off (`git commit -s`) on every commit. Manual admission does not require cryptographic commit signing. +- First-time external contributors must be vouched. See the [Vouch System](AGENTS.md#vouch-system). Flow: -1. Open the PR. The vouch check confirms you are vouched (otherwise the PR is auto-closed). -2. copy-pr-bot does not mirror forks automatically. A maintainer reviews the diff and comments `/ok to test ` with your latest commit SHA. -3. After `/ok to test`, copy-pr-bot mirrors to `pull-request/`. From here the flow is identical to internal PRs: `Required CI Gates` verifies the mirror and required push workflows, and maintainers apply the E2E label when the extra suites are needed. +1. Open the PR. The vouch check confirms first-time external contributors are vouched (otherwise their PRs are auto-closed). +2. If copy-pr-bot does not mirror it automatically, a listed vetter reviews the diff and comments `/ok to test ` with the latest commit SHA. Fork location alone does not determine whether a PR is mirrored automatically. +3. After `/ok to test`, copy-pr-bot mirrors to `pull-request/`. From here the flow is identical to automatically admitted PRs: `Required CI Gates` verifies the mirror and required push workflows, and maintainers apply the E2E label when the extra suites are needed. 4. When the PR is ready to merge, maintainers add it to the merge queue so the queued integration state is tested before it reaches `main`. -Important: every new commit you push requires another `/ok to test ` from a maintainer before push-based CI will run on it. If a label is applied while the mirror is stale, `E2E Label Help` will post a comment explaining what's needed. +Important: if a PR requires manual admission, every new commit needs another `/ok to test ` from a listed vetter before push-based CI will run on it. If a label is applied while the mirror is stale, `E2E Label Help` will post a comment explaining what's needed. ## Merge queue diff --git a/architecture/build.md b/architecture/build.md index a447b03d60..87cd8deb24 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -372,6 +372,10 @@ the release tag. Required checks run on GitHub Actions. Pull-request workflows that use NVIDIA self-hosted runners trigger from copy-pr-bot mirror branches, so trusted PRs are mirrored into `pull-request/` branches before those workflows run. `main` also uses GitHub merge queue so the final queued integration commit is validated before it merges. +For PRs that need manual admission, copy-pr-bot accepts `/ok to test ` +only from the explicit `vetters_override` list in `.github/copy-pr-bot.yaml`. +This list is maintained separately from the bot's automatic PR trust policy. + The high-level CI model: 1. PR-context gate jobs publish required statuses for the PR head commit. From f213e9a49bd7740deee3330f409ccf3ecb26a54c Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 24 Sep 2026 23:43:31 +0000 Subject: [PATCH 4/7] docs(middleware): reorganize and expand middleware guides (#3636) * docs(extensibility): reorganize extensibility and middleware guides Add an extensibility overview that introduces the extension protocol and links every extension point, and move extension caller authentication to a shared page used by middleware and gateway interceptors. Split the supervisor middleware guide into an overview, a Configure and Operate guide, and a Middleware Operations reference for service authors. The overview explains when to use middleware, shows where it runs, lists current limitations, and defines the service contract. Configure and Operate covers policy attachment, service registration, failure behavior, and observability. Middleware Operations describes HTTP request, HTTP response, and WebSocket message operations with shared inputs and results, per-operation diagrams, and detail accordions. Pin page slugs so links resolve, redirect the replaced dev middleware URL, and update reference and architecture links. Signed-off-by: Piotr Mlocek * docs(middleware): clarify navigation and service contracts Signed-off-by: Piotr Mlocek * docs(middleware): drop obsolete dev URL redirect Signed-off-by: Piotr Mlocek --------- Signed-off-by: Piotr Mlocek --- architecture/sandbox-limits.md | 5 +- architecture/sandbox.md | 5 +- docs/about/overview.mdx | 2 +- docs/extensibility/drivers.mdx | 2 +- .../extension-authentication.mdx | 72 +++++ docs/extensibility/extension-negotiation.mdx | 3 +- docs/extensibility/gateway-interceptors.mdx | 16 +- docs/extensibility/overview.mdx | 43 +++ docs/extensibility/supervisor-middleware.mdx | 246 ----------------- .../supervisor-middleware/configure.mdx | 199 ++++++++++++++ .../supervisor-middleware/index.mdx | 111 ++++++++ .../supervisor-middleware/operations.mdx | 251 ++++++++++++++++++ docs/index.yml | 16 +- docs/reference/gateway-config.mdx | 2 +- docs/reference/policy-schema.mdx | 2 +- docs/sandboxes/policies.mdx | 2 +- 16 files changed, 707 insertions(+), 270 deletions(-) create mode 100644 docs/extensibility/extension-authentication.mdx create mode 100644 docs/extensibility/overview.mdx delete mode 100644 docs/extensibility/supervisor-middleware.mdx create mode 100644 docs/extensibility/supervisor-middleware/configure.mdx create mode 100644 docs/extensibility/supervisor-middleware/index.mdx create mode 100644 docs/extensibility/supervisor-middleware/operations.mdx diff --git a/architecture/sandbox-limits.md b/architecture/sandbox-limits.md index 049aa5d99d..4a245a5ed9 100644 --- a/architecture/sandbox-limits.md +++ b/architecture/sandbox-limits.md @@ -72,8 +72,9 @@ budgets as new activity. Middleware also validates every non-body envelope component. Important examples include 64 KiB service config, 4 KiB request context, 32 KiB target data, 128 request headers totaling 64 KiB, 64 header mutations, 32 findings per stage, -and 64 metadata entries. The detailed external contract lives in -[Supervisor Middleware](../docs/extensibility/supervisor-middleware.mdx). +and 64 metadata entries. The external contract lives in +`proto/supervisor_middleware.proto`, with service-author guidance in the +[supported middleware operations](../docs/extensibility/supervisor-middleware/operations.mdx). The work semaphore bounds aggregate buffered middleware input to approximately `32 × 4 MiB`, plus bounded envelope and parser overhead. It is a concurrency diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 26a446e4f0..7d5fdb23bc 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -442,8 +442,9 @@ against body-aware L7 policy before later stages or the upstream can observe them. Requests, results, chain length, execution time, and diagnostics are bounded; external free-form diagnostic text is not exposed in responses or security logs. See -[Supervisor Middleware](../docs/extensibility/supervisor-middleware.mdx) for -configuration and protocol details. +[Supervisor Middleware](../docs/extensibility/supervisor-middleware/index.mdx) for +an introduction, or the [configuration guide](../docs/extensibility/supervisor-middleware/configure.mdx) +for service registration and policy attachment. Inference providers use the same egress path as other external services. An attached provider profile contributes endpoint and binary policy. The proxy diff --git a/docs/about/overview.mdx b/docs/about/overview.mdx index 3e8632080f..82be4bbe48 100644 --- a/docs/about/overview.mdx +++ b/docs/about/overview.mdx @@ -11,7 +11,7 @@ position: 1 NVIDIA OpenShell is an open-source runtime for executing autonomous AI agents in sandboxed environments with kernel-level isolation. It combines sandbox runtime controls and a declarative YAML policy so teams can run agents without giving them unrestricted access to local files, credentials, and external networks. -New in OpenShell 0.1.0 is a [stable release cadence](/about/installation#release-cadence), a stronger [security model](/about/how-it-works), and an expanded [extension surface](/extensibility/extension-negotiation), along with much more. +New in OpenShell 0.1.0 is a [stable release cadence](/about/installation#release-cadence), a stronger [security model](/about/how-it-works), and an expanded [extension surface](/extensibility/overview), along with much more. See our [upgrade guide](/upgrade/0-1-0) for everything that's changed. diff --git a/docs/extensibility/drivers.mdx b/docs/extensibility/drivers.mdx index c988b651a6..1f6fdc0800 100644 --- a/docs/extensibility/drivers.mdx +++ b/docs/extensibility/drivers.mdx @@ -36,5 +36,5 @@ Configure the active credential driver in the Drivers exchange peer metadata with the gateway and advertise their extension family capability. Upgrade both peers together when the protocol version -changes. Refer to [Extensibility Overview](/extensibility/extension-negotiation) +changes. Refer to [Extension Protocol Negotiation](/extensibility/extension-negotiation) for the negotiation contract. diff --git a/docs/extensibility/extension-authentication.mdx b/docs/extensibility/extension-authentication.mdx new file mode 100644 index 0000000000..3861c99aad --- /dev/null +++ b/docs/extensibility/extension-authentication.mdx @@ -0,0 +1,72 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Extension Authentication" +sidebar-title: "Authentication" +slug: "extensibility/extension-authentication" +description: "Verify that calls to your gateway interceptor or supervisor middleware service come from your OpenShell gateway." +keywords: "OpenShell Extensions, Extension Authentication, JWT, JWKS, Audience, Gateway Interceptors, Supervisor Middleware" +--- + +When the gateway has JWT signing configured, OpenShell attaches a short-lived bearer token to every call it makes to a [gateway interceptor](/extensibility/gateway-interceptors) or [supervisor middleware](/extensibility/supervisor-middleware) service. Your service validates this token to confirm that the call comes from your gateway or one of its sandboxes. + +## Tokens OpenShell Sends + +Each token is an EdDSA-signed JWT scoped to one service registration: + + + `openshell-gateway:`. + + + + The registration's `audience`. Defaults to `urn:openshell:extension:interceptor:` or `urn:openshell:extension:middleware:`. + + + + `gateway` for calls from the gateway, or `supervisor` for middleware calls from a sandbox supervisor. + + + + The calling sandbox. Present only when `caller_kind` is `supervisor`. + + +Authenticated network services use `https://` endpoints. OpenShell verifies the service certificate against platform trust roots, or against the private CA in the registration's `tls_ca_cert_path`, and always checks the hostname. + +## Establish Trust in the Gateway + +Before your service can validate tokens, configure it with three values from the gateway operator: + +- The gateway URL. +- The gateway ID. Tokens use `openshell-gateway:` as their issuer. +- The gateway's public signing key or JWKS. + +Configure these values directly instead of discovering them from the gateway. A key fetched from an unverified source does not prove which gateway it belongs to. + +To pick up new keys later, fetch `/.well-known/openid-configuration` from the configured gateway URL over TLS, then fetch the keys from its `jwks_uri`. This document looks like OIDC discovery, but its `issuer` is `openshell-gateway:` rather than the gateway URL. Compare the token's `iss` with that value, not with the URL. + +## Validate Each Token + +Cache keys by `kid` and check that: + +- `typ` is exactly `openshell-ext+jwt`. +- `alg` is `EdDSA`. Pin the algorithm instead of reading it from the token. +- The signature, issuer, exact audience, and expiry are valid. +- `caller_kind` is one your service accepts, and `sandbox_id` matches your expectations when your service scopes behavior per sandbox. + +OpenShell reuses a token until it rotates, so do not reject a repeated `jti` as a replay. + +## Confirm the Audience at Startup + +Return the audience your service expects in the `expected_audience` field of its `Describe` manifest. After authentication succeeds, the gateway compares this value with the operator-configured `audience` and refuses to start when they differ. A strict service may reject a token with the wrong audience before returning its manifest, in which case gateway startup reports an authentication failure. Leave the field empty to skip this check. + +## Run Without Authentication + +Set `allow_insecure_transport = true` on a registration to use a plaintext `http://` endpoint. OpenShell then sends no token, and your service cannot distinguish OpenShell from any other client that can reach it. The gateway logs a warning for the registration at every startup. + +Use this setting for local development, or on a network that already authenticates callers. + +## Current Limitations + +- Tokens are bearer credentials. A captured token remains valid until it expires unless your service adds proof of possession or request binding. +- Extension tokens share a signing key with other tokens the gateway issues, so you cannot rotate or revoke extension credentials independently. +- mTLS client authentication and overlapping signing-key rotation are not available. diff --git a/docs/extensibility/extension-negotiation.mdx b/docs/extensibility/extension-negotiation.mdx index eca4d6a47e..6ffbeea9b0 100644 --- a/docs/extensibility/extension-negotiation.mdx +++ b/docs/extensibility/extension-negotiation.mdx @@ -2,7 +2,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Extension Protocol Negotiation" -sidebar-title: "Overview" +sidebar-title: "Protocol Negotiation" +slug: "extensibility/extension-negotiation" description: "Implement version and capability negotiation for OpenShell extensions." keywords: "OpenShell Extensions, Protocol Version, Capabilities, Version Skew, Migration" --- diff --git a/docs/extensibility/gateway-interceptors.mdx b/docs/extensibility/gateway-interceptors.mdx index 69c82b46d1..96b091dbca 100644 --- a/docs/extensibility/gateway-interceptors.mdx +++ b/docs/extensibility/gateway-interceptors.mdx @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 title: "Gateway Interceptors" sidebar-title: "Gateway Interceptors" +slug: "extensibility/gateway-interceptors" description: "Extend OpenShell gateway operations with deployment-specific governance and business logic." keywords: "Generative AI, Cybersecurity, AI Agents, Gateway Interceptors, Extensibility, Governance" --- @@ -10,7 +11,7 @@ Gateway interceptors let operators add deployment-specific governance to OpenShe See the [governance interceptor example](https://github.com/NVIDIA/OpenShell/tree/main/examples/governance-interceptor) for a complete service that vends provider profiles, applies a signed policy to new sandboxes, and rejects attempts to weaken that policy. -## Choose Gateway Interceptors +## When to Use Gateway Interceptors Use a gateway interceptor when an external service needs to govern gateway API operations. For example, an interceptor can: @@ -76,7 +77,7 @@ Start the interceptor before the gateway, then register it in gateway TOML: [[openshell.gateway.interceptors]] name = "policy-governance" grpc_endpoint = "https://governance.example:18081" -tls_ca_cert_path = "/etc/openshell/governance-ca.pem" +tls_ca_cert_path = "/path/to/governance-ca.pem" audience = "urn:example:governance" order = 10 failure_policy = "fail_closed" @@ -96,13 +97,7 @@ phases = ["validate"] The gateway supports `http://`, `https://`, and `unix://` interceptor endpoints. When gateway JWT signing is configured, authenticated network interceptors use `https://`; Unix sockets remain available for local integrations. HTTPS uses platform trust roots unless `tls_ca_cert_path` supplies a private CA, and normal hostname verification remains enabled. The gateway calls `Describe` and builds an immutable execution plan during startup. An unavailable service, invalid manifest, missing credential, or unauthorized configured binding prevents the gateway from starting. -The gateway attaches a short-lived EdDSA bearer token to `Describe`, `Evaluate`, and provider-profile snapshot calls. The token uses the configured `audience` (defaulting to `urn:openshell:extension:interceptor:`) and `caller_kind: gateway`. - -Return your expected audience in the `expected_audience` field of your `Describe` manifest. After authenticated `Describe` succeeds, the gateway compares the advertised value with its operator-configured audience and refuses to start when they differ. This is a post-authentication consistency assertion, not audience discovery: a strict verifier may reject an incorrect audience before returning the manifest, in which case startup reports an authentication failure. Leave the field empty to skip the consistency check. - -Provision the trusted gateway URL, expected gateway ID, and public key or JWKS through the deployment. This operator-provisioned key material is the authoritative cold-start trust anchor. The expected issuer is exactly `openshell-gateway:`; fetching JWKS does not establish that identity by itself. After initial trust is established, `GET /.well-known/openid-configuration` and its `jwks_uri` provide steady-state key refresh and operational convenience. The document is OIDC-shaped rather than OIDC-compliant because `issuer` is the gateway identity rather than the serving URL; compare `iss` against the configured value and fetch updates only over authenticated TLS at the trusted gateway URL. Pin `alg` to `EdDSA`, require `typ` to be exactly `openshell-ext+jwt`, and validate `kid`, signature, expected issuer, exact audience, positive expiry, and caller kind. - -Set `allow_insecure_transport = true` on an interceptor to keep a plaintext `http://` endpoint working with no credential attached. The gateway logs a warning naming the interceptor at every startup, and the service cannot distinguish the gateway from any other client that can reach it. +When gateway JWT signing is configured, the gateway authenticates every call to the interceptor with a short-lived token. [Extension Authentication](/extensibility/extension-authentication) describes how your service validates it. Set `allow_insecure_transport = true` to use a plaintext `http://` endpoint without authentication, for local development or on a network that already authenticates callers. Registration is static. Restart the gateway after adding, removing, or changing an interceptor. See [Gateway Configuration](/reference/gateway-config#gateway-interceptors) for the complete field reference. @@ -175,6 +170,5 @@ The gateway emits structured evaluation logs containing the interceptor name, bi - Only explicitly allowlisted unary write RPCs are interceptable. New gateway RPCs are non-interceptable until added to the allowlist. - `current_state` is available only in the `validate` contract. The gateway does not yet populate it with method-specific state. - Registration changes require a gateway restart. -- mTLS client authentication, service health checks, runtime registration, and overlapping signing-key rotation are not available. -- Extension tokens and sandbox-to-gateway tokens are signed by the same key, separated by audience and `typ`. The extension credential path cannot yet be rotated or revoked independently of sandbox admission. +- Service health checks and runtime registration are not available. [Extension Authentication](/extensibility/extension-authentication#current-limitations) lists authentication limitations. - Interceptors cannot receive or mutate protobuf fields marked secret. diff --git a/docs/extensibility/overview.mdx b/docs/extensibility/overview.mdx new file mode 100644 index 0000000000..44c9da6f15 --- /dev/null +++ b/docs/extensibility/overview.mdx @@ -0,0 +1,43 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Extensibility" +sidebar-title: "Overview" +slug: "extensibility/overview" +description: "Extend OpenShell with custom traffic checks, gateway governance, drivers, and isolation backends." +keywords: "OpenShell Extensions, Protocol Negotiation, Supervisor Middleware, Gateway Interceptors, Compute Drivers, Credential Drivers, Isolation Backends" +--- + +OpenShell has several extension points that let you connect your own services and runtimes without changing OpenShell. Choose an extension point based on what you need to control, then implement the shared extension protocol that every extension service uses. + +## Extension Protocol + +Before OpenShell uses an extension service, both peers exchange their protocol version and supported capabilities. This lets OpenShell and your service detect incompatible versions at startup instead of failing on live traffic. [Extension Protocol Negotiation](/extensibility/extension-negotiation) describes the metadata to exchange and the version-skew policy. + +Extension services reachable over the network also need to know that a call comes from your OpenShell gateway. [Extension Authentication](/extensibility/extension-authentication) describes the tokens OpenShell attaches and how your service validates them. + +## Extension Points + + + + + +Check or change the content an agent sends and receives over the network, such as redacting API tokens from outgoing requests or blocking prohibited content. + + + + +Enforce rules on how people and applications manage OpenShell resources, such as applying an approved policy to new sandboxes or auditing completed operations. + + + + +Connect the gateway to the runtimes that host sandbox workloads and to the stores that hold provider credentials. + + + + +Understand the runtime boundary the supervisor uses to launch and control processes inside a sandbox. + + + diff --git a/docs/extensibility/supervisor-middleware.mdx b/docs/extensibility/supervisor-middleware.mdx deleted file mode 100644 index 2d03b8de77..0000000000 --- a/docs/extensibility/supervisor-middleware.mdx +++ /dev/null @@ -1,246 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Supervisor Middleware" -sidebar-title: "Supervisor Middleware" -description: "Configure and operate built-in and operator-run middleware for sandbox HTTP requests and WebSocket messages." -keywords: "Generative AI, Cybersecurity, AI Agents, Supervisor Middleware, Extensibility, Request Filtering" ---- - -Supervisor middleware adds ordered processing stages to allowed HTTP and WebSocket egress. Middleware runs after network and L7 policy admit traffic and before OpenShell injects provider credentials. A stage can allow or deny an HTTP request or client WebSocket text message, replace its payload, add approved HTTP headers, and report audit-safe findings. - -Middleware selection is independent of the network policy rule that admitted the request. OpenShell matches middleware by destination host, so the same middleware applies consistently across broad, specific, user-authored, and provider-derived network policies. - -## Request Flow - -For each inspected HTTP request, the supervisor: - -1. Evaluates network and L7 policy. -2. Selects middleware whose host selectors match the admitted destination. -3. Buffers the request body using the largest body limit in the selected chain. -4. Runs matching middleware by ascending `order`. Policy validation rejects duplicate order values. -5. Re-checks body-aware protocol policy (GraphQL, JSON-RPC, MCP) after each stage that replaces the body. Every middleware receives a payload the policy admits, and a transformation cannot smuggle a denied or unparseable operation to a later stage or the upstream. -6. Applies allowed transformations, injects provider credentials, and forwards the request. - -Response middleware advertising `HTTP_RESPONSE/PRE_RETURN` inspects the final upstream response before OpenShell returns it to the workload. Its preflight includes upstream `Content-Length`, `Content-Encoding`, and `Content-Range` fields as read-only metadata, except when `Connection` nominates the field as hop-by-hop. Middleware cannot write or remove these fields. OpenShell computes downstream framing separately and emits the final `Content-Length` or `Transfer-Encoding` exactly once after middleware processing. - -For an RFC 6455 upgrade over `ws://` or `wss://`, the supervisor first finds every host-matched attachment, then selects only implementations that advertise `WEBSOCKET_MESSAGE/PRE_CREDENTIALS`. It opens one ordered, phase-specific `EvaluateWebSocketSession` stream per selected stage. OpenShell sends `WebSocketSessionEvent` values, while the service returns `WebSocketSessionEventResult` values only for preflight and message events; session start and end are notifications. Future upstream-to-client inspection uses the same RPC with `PRE_RETURN`; an implementation that advertises both phases receives two independent streams for the WebSocket session. An attachment without the selected binding can still inspect the HTTP upgrade request when it advertises the HTTP binding, but it is not a failed WebSocket stage. OpenShell allows post-upgrade traffic and emits an informational `binding_not_selected` coverage event for that attachment. - -1. A preflight before the upgrade is sent upstream. The stage chooses `INSPECT`, voluntary `SKIP`, or authoritative `DENY` and may return a bounded diagnostic reason, stable reason code, findings, and metadata. OpenShell runs selected preflights concurrently; any `DENY` rejects the upgrade regardless of `on_error`. -2. A session-start event after the upstream accepts the upgrade, including the negotiated subprotocol. -3. Complete client-to-upstream text messages in sequence order. OpenShell reassembles fragmented messages and decompresses negotiated `permessage-deflate` messages before evaluation. -4. A best-effort session-end event when the stage stream remains writable. OpenShell attempts at most one terminal event for each opened stream, including streams opened during a preflight that rejects the upgrade before session start. It then half-closes the request stream and briefly drains the response stream so the terminal event can leave the local transport before the RPC closes. Middleware services should finish their response stream after the request stream reaches EOF. - -The protobuf represents each logical message with a `text` or `binary` payload variant. Text uses the protobuf `string` type, so invalid UTF-8 cannot enter the middleware contract. Results use an optional matching replacement variant: absence preserves the input, while presence represents a replacement even when its content is empty. OpenShell rejects attempts to change the message type. Allowed replacements are re-framed, re-compressed when required, and forwarded. Binary messages, control frames, and upstream-to-client traffic remain uninspected. Binary messages pass through under both `on_error` modes. For each active selected stage, OpenShell emits an informational `unsupported_message_type` coverage event and advances the session-global sequence; the next text message can therefore reach the stage with a valid sequence gap. - -The network supervisor reserves process-wide assembly capacity before buffering every parsed WebSocket text message, even when no middleware is selected. At most 32 assemblies run while 64 additional callers wait without buffering payload bytes. When both bounds are full, OpenShell closes the WebSocket with code `1013` before reading the new message payload. A text message may contain at most 4,096 fragments, must make input progress within 30 seconds, and must finish assembly within 2 minutes. Forwarding the completed text frame must finish within another 2 minutes. The assembly budget lasts for the supervisor process lifetime, so policy reloads do not reset its capacity. - -Active middleware sessions additionally reserve shared middleware capacity before buffering WebSocket text, and HTTP middleware reserves the same capacity before buffering request bodies; at most 32 evaluations run and 64 additional unbuffered callers wait for capacity. When both middleware bounds are full, OpenShell sheds an HTTP request with `503 Service Unavailable` before reading its body. Persistent middleware streams use a separate process-wide budget of 32 sessions. WebSocket session admission does not wait: if the budget is full, OpenShell applies each selected config's `on_error` behavior before opening a stream. - -Because each transformed body is re-checked before the next stage runs, a middleware hook always receives a request that satisfies the sandbox policy. A stage whose output the policy rejects stops the chain; under `enforcement: audit` the rejection is logged and the request proceeds. - -If post-transformation policy evaluation itself fails, OpenShell denies the request and emits a high-severity detection finding. This failure is separate from middleware `on_error` because the middleware completed successfully; the sandbox policy could not validate its output. - -Middleware receives the request before credential injection. Operator-run services cannot inspect OpenShell-managed credentials. Middleware-visible request headers are delivered in wire order and repeated header names are preserved as separate entries. OpenShell filters credential, routing, framing, and hop-by-hop headers before invoking middleware. It rejects malformed request headers and unsupported transfer-coding sequences before middleware or policy dispatch. Headers named by a request's `Connection` field are omitted from middleware input and removed before forwarding, except for the validated WebSocket upgrade pair. - -The request context identifies the originating sandbox to operator-run services. It carries the sandbox ID (`sandbox_id`), the sandbox name (`sandbox`), and the workspace (`workspace`), letting audit and approval interfaces show a human-readable name and its workspace instead of an opaque ID. `sandbox` and `workspace` are for display and logging only: names are workspace-scoped and may be reused for different sandbox instances, so services must use `sandbox_id` for authorization, persistence, durable correlation, and identity. `sandbox_id` is always present on middleware requests. `sandbox` and `workspace` are best-effort: a supervisor that cannot resolve a value, or an older supervisor that predates a field, sends an empty string. Services should fall back to the sandbox ID when the name or workspace is empty. - -## Choose a Middleware Type - -| Type | Registration | Payload limit | Deployment | -| --- | --- | --- | --- | -| Built-in | None | Defined by OpenShell | Runs inside the supervisor | -| Operator-run service | Required in gateway TOML | Set by the operator, up to the service capability | Runs as a separate service reachable by the gateway and supervisors | - -`openshell/regex` is an example built-in middleware. It replaces only simple, self-contained token patterns in UTF-8 HTTP bodies and client WebSocket text messages; the initial pattern recognizes `sk-` tokens. It does not infer values from keyword assignments such as JSON `password` fields. This best-effort text transformation is not parser-aware and does not guarantee that it will detect or fully remove sensitive values. Its `config` accepts one field, `mode: redact`, which is also the default when the field is omitted. Unknown config fields and non-string values are rejected at policy validation. Custom expressions are not configurable yet. - -Operator-run services expose bindings for supported operation and phase pairs. A binding is identified by its operation and phase. V1 supports `HttpRequest/pre_credentials` and `WebSocketMessage/pre_credentials`; a service may expose either or both. Policies attach the complete middleware by its operator-owned gateway registration name. - -## Register a Middleware Service - -Start an operator-run service before starting the gateway, then add a registration to the local gateway TOML: - -```toml -[[openshell.supervisor.middleware]] -name = "local-content-guard" -grpc_endpoint = "https://content-guard.example:50051" -tls_ca_cert_path = "/etc/openshell/content-guard-ca.pem" -audience = "urn:example:content-guard" -max_payload_bytes = 262144 -timeout = "500ms" -``` - -| Field | Description | -| --- | --- | -| `name` | Operator-owned registration name used by policy attachments and diagnostics. Names must be unique, and `openshell/` is reserved for built-ins. | -| `grpc_endpoint` | Service address reachable from both the gateway and sandbox supervisors. Authenticated extensions use TLS `https://`. | -| `tls_ca_cert_path` | Optional PEM trust roots for a private HTTPS service. Custom roots replace platform roots and retain hostname verification. | -| `audience` | Exact audience expected by the service. Defaults to `urn:openshell:extension:middleware:`. | -| `allow_insecure_transport` | Opt this registration out of extension authentication, permitting a plaintext `http://` endpoint with no bearer credential. Defaults to `false`. Development and trusted-network deployments only. | -| `max_payload_bytes` | Shared operator limit applied to inspectable logical payloads across every binding exposed by the service, up to the 4 MiB platform maximum. It caps HTTP bodies and complete WebSocket text messages. | -| `timeout` | Optional service-wide RPC timeout using an integer with an `ms` or `s` suffix. Defaults to `500ms`; valid values range from `10ms` through `30s`. | - -Each binding returned by `Describe` may advertise a shorter `timeout` using the same syntax and bounds. The operator-configured service timeout is a ceiling: OpenShell uses the smaller of the binding and service values. An omitted binding timeout inherits the service setting, and an omitted service setting uses the 500 ms platform default. OpenShell rejects an invalid timeout before accepting the manifest. The operator-configured service timeout applies to `Describe` and `ValidateConfig`. The effective binding timeout applies only to `EvaluateHttpRequest`, WebSocket preflight, and each WebSocket message. WebSocket streams have no connection-wide deadline. - -The gateway connects to every registered service and verifies its capabilities before accepting traffic. Gateway startup fails when a service is unavailable, reports an invalid capability, or exposes more than one binding for the same operation and phase. The manifest `name` is diagnostic metadata and does not need to match the operator registration name. Operator-run registration names cannot claim the reserved `openshell/` namespace. - -`MiddlewareDescribeRequest` carries the caller's common protocol metadata, and `MiddlewareManifest.extension` returns the service's protocol and implementation metadata. This replaces the former empty `Describe` request. Regenerate service bindings and return protocol `1.0` plus `openshell.supervisor-middleware.contract`. Legacy gateways and current middleware cannot interoperate, so upgrade both peers during a coordinated outage. See [Extension Protocol Negotiation](/extensibility/extension-negotiation) for the skew policy and migration sequence. - -Registration is static. Restart the gateway after adding, removing, or changing a service. See [Gateway Configuration](/reference/gateway-config#supervisor-middleware-services) for the complete gateway TOML context. - -### Authenticate OpenShell Callers - -When gateway JWT signing is configured, OpenShell attaches a short-lived EdDSA bearer token to every remote middleware RPC. Gateway calls use `caller_kind: gateway`; sandbox supervisor calls use `caller_kind: supervisor` and include the sandbox ID. Supervisors request credentials by registration name through `RefreshSandboxToken`. The gateway derives the audience from operator-owned configuration and authorizes each name against the sandbox's effective policy. - -Return your expected audience in the `expected_audience` field of your `Describe` manifest. After authenticated `Describe` succeeds, OpenShell compares the advertised value with its operator-configured audience and refuses to start when they differ. This is a post-authentication consistency assertion, not audience discovery: a strict verifier may reject an incorrect audience before returning the manifest, in which case startup reports an authentication failure. Leave the field empty to skip the consistency check. - -Provision the trusted gateway URL, expected gateway ID, and public key or JWKS through the deployment. This operator-provisioned key material is the authoritative cold-start trust anchor. The expected issuer is exactly `openshell-gateway:`; fetching JWKS does not establish that identity by itself. After initial trust is established, `GET /.well-known/openid-configuration` and its `jwks_uri` provide steady-state key refresh and operational convenience. The document is OIDC-shaped rather than OIDC-compliant: `issuer` is the gateway identity, not the URL serving the document, so compare `iss` against the configured value and fetch updates only over authenticated TLS at the trusted gateway URL. - -Cache keys by `kid`. Validate, at minimum: - -- `typ` is exactly `openshell-ext+jwt`. Extension tokens and sandbox-to-gateway bootstrap tokens share a signing key and differ only in audience; this header is a second, independent discriminator. -- `alg` is pinned to `EdDSA`. Never select the algorithm from the token. -- Signature, expected issuer, exact audience, and positive expiry. -- `caller_kind`, and the sandbox identity when your service scopes behavior per sandbox. - -A sandbox-to-gateway JWT is not an extension credential even though both token types use the same signing key. - -Each token carries a unique `jti` that identifies that token instance for correlation and future explicit revocation. OpenShell reuses a token across calls until rotation and does not track `jti`, so rejecting a repeated `jti` would reject legitimate requests. Per-request replay resistance requires a request nonce or signature, channel binding, or another proof-of-possession mechanism. - -### Run Without Extension Authentication - -Set `allow_insecure_transport = true` on a registration to keep a plaintext `http://` endpoint working. OpenShell then attaches no credential to that service, supervisors do not request one, and the gateway refuses to mint one if asked. The gateway logs a warning naming the registration at every startup. - -The service cannot distinguish OpenShell from any other client that can reach it. Use this only where the network already provides that guarantee, and prefer `https://` everywhere else. - -## Apply Middleware with Policy - -Add middleware configs to the top-level `network_middlewares` map. Each key is the policy-local config name: - -```yaml -network_middlewares: - regex-redactor: - name: Redact API tokens - middleware: openshell/regex - order: 10 - config: - mode: redact - on_error: fail_closed - endpoints: - include: ["*.example.com"] - exclude: ["trusted.example.com"] -``` - -Each config has a stable policy-local identity from its map key, an optional human-readable `name` that defaults to that key, a built-in or operator-owned registration name in `middleware`, an integer `order`, implementation-owned `config`, failure behavior, and host selectors. The optional name does not replace the map key for attachment or future keyed updates. A policy accepts at most 10 middleware configs. - -`include` selects destination hosts. `exclude` takes precedence and removes hosts from that selection. Each config accepts at most 32 combined include and exclude patterns. Matching is case-insensitive and uses the same exact-host and DNS glob behavior as network policy endpoints: `*` matches exactly one DNS label, `**` matches one or more labels, and intra-label patterns like `*-api.example.com` work. Brace alternates such as `{prod,staging}` are rejected at validation; list each host pattern separately. - -Matching configs run once each by ascending `order`; lower values run first. Order values must be unique across the complete policy, even when endpoint selectors do not overlap. The default order is `0`, so policies with multiple configs normally set explicit values. Different map keys may attach the same middleware and run as separate stages. Map keys are structurally unique. Runtime selection defensively rejects chains with more than 10 stages. - -See [Policy Schema](/reference/policy-schema#network-middleware) for the complete field reference. - -## Configure Failure Behavior - -`on_error` controls what happens after an operation binding is selected and middleware is unavailable, rejects its configuration, returns an invalid result, or exceeds the selected binding's payload limit. It does not turn an unadvertised operation or an unsupported WebSocket message class into a middleware failure. - -| Value | Behavior | -| --- | --- | -| `fail_closed` | Denies the HTTP request or closes the WebSocket when the stage fails. This is the default. | -| `fail_open` | Skips the failed HTTP stage. For a broken WebSocket stage stream, disables that stage for the rest of the connection and continues the remaining chain. | - -A valid upstream response can exceed the middleware envelope limits or contain header bytes that the middleware protocol cannot represent. OpenShell relays the original response when every selected response stage uses `fail_open`; any selected `fail_closed` stage causes the canonical 502 delivery failure. Malformed or unsafe HTTP does not qualify for this bypass. - -Use `fail_open` only when bypassing the middleware preserves the intended security policy. OpenShell emits a detection finding when a failed stage is bypassed and a separate state-change finding when a WebSocket stage is disabled for the session. - -Capability coverage is separate from failure handling. A host-matched HTTP-only attachment does not join the WebSocket chain, regardless of `on_error`. Binary messages are outside the V1 text-message binding and pass through even when a selected stage is `fail_closed`. OpenShell records both states as informational coverage events so operators do not mistake pass-through traffic for inspected traffic. If a deployment requires all WebSocket message classes to be inspected, V1 cannot express that requirement. - -An explicit deny decision always stops the chain and denies the request or WebSocket upgrade, regardless of `on_error`. A WebSocket preflight `DENY` is a successful policy decision, not a middleware failure; OpenShell rejects the upgrade before upstream contact and ends each still-writable stream opened by a successful preflight decision with `MIDDLEWARE_DENIAL`. The HTTP response uses `error: middleware_denied`, identifies the policy-local middleware config, and omits policy-advisor remediation because the network and L7 allow rules already matched. OpenShell never copies the free-form middleware `reason` into the response or security logs. HTTP results, WebSocket preflight decisions, and WebSocket message results can instead return an optional stable `reason_code`: 1–64 bytes, starting with a lowercase ASCII letter and containing only lowercase ASCII letters, digits, and underscores. Invalid codes make the result a middleware failure governed by `on_error`. Preflight findings and metadata use the same bounds and audit-safe handling as message results. - -```json -{ - "error": "middleware_denied", - "detail": "Request rejected by configured middleware", - "policy": "api-policy", - "middleware": "prototype-content-guard", - "reason_code": "content_match" -} -``` - -A failed `fail_closed` stage uses `error: middleware_failed` and a platform-owned `detail`. It also omits `rule_missing`, `next_steps`, and `agent_guidance`: the failure did not result from a missing network or L7 policy rule, and changing policy cannot repair it. Runtime diagnostic text is available only through sanitized operator telemetry. - -Middleware decisions are enforced regardless of the endpoint's `enforcement` mode. `enforcement: audit` applies to an endpoint's network and L7 policy rules and does not bypass middleware: a middleware deny, or a failed `fail_closed` stage, blocks the request even on an audit endpoint. A middleware service that needs to observe traffic without blocking should return an allow decision with findings, which OpenShell emits as detection findings. - -## Set Payload Limits - -Every middleware binding declares the largest logical payload or replacement it supports through `max_payload_bytes`. For `HTTP_REQUEST`, that payload is one request body. For `WEBSOCKET_MESSAGE`, it is one complete message rather than the whole session. - -- Built-in middleware uses its OpenShell-defined limit. -- Each operator-run registration sets one `max_payload_bytes` ceiling no higher than any binding's advertised `max_payload_bytes` capability. -- A selected chain buffers using its largest stage limit, so every stage that can process the body receives it. -- The same per-stage limit applies to request bodies and replacement bodies. - -The gateway rejects a registration whose operator limit exceeds the service capability or the 4 MiB platform maximum instead of silently clamping it. OpenShell also bounds the non-payload protobuf components: 64 KiB for service config, 4 KiB for request context, 32 KiB for the target, and 128 request header lines totaling at most 64 KiB encoded. Results allow a 4 KiB discarded free-form reason, a 64-byte validated reason code, 64 header mutations totaling at most 64 KiB encoded, 32 findings of at most 4 KiB encoded each, and 64 metadata entries totaling at most 32 KiB. Middleware gRPC servers should configure request and response message limits to at least 4 MiB plus 293 KiB so every platform-valid envelope fits. - -At request time, exceeding a selected stage's limit is a middleware failure for that stage alone and follows that config's `on_error` behavior; other stages in the chain still run against their own limits. OpenShell can apply `fail_open` to an oversized `Content-Length` before consuming body bytes. A chunked body can cross the limit only after bytes have been consumed, so OpenShell denies that request because it cannot safely resume the original stream. - -For a WebSocket binding, `max_payload_bytes` covers complete client text messages and replacements. Exceeding a selected stage's effective text-message limit follows that stage's `on_error`. The 4 MiB parsed-text platform cap and other protocol-safety limits are independent of middleware failure policy. Binary messages are not delivered to middleware, so the operator ceiling does not become a binary relay limit; individual raw binary frames retain the 16 MiB relay-safety bound. Oversized parsed text closes the connection with code `1009`; invalid UTF-8 uses `1007`; protocol errors use `1002`; middleware or policy denials use `1008`; and policy reload uses `1012`. - -## Mutate Request Headers - -A middleware result can return ordered header mutations before OpenShell injects credentials. A `write` mutation adds a value when the case-insensitive header name is absent and selects one behavior when it is already present: - -- `append` adds another field value. -- `overwrite` removes every existing value before adding the new value. -- `skip` leaves existing values unchanged. - -A `remove` mutation removes every value for a case-insensitive header name. OpenShell applies each successful stage's mutations before invoking the next middleware, so later stages observe the accumulated header state. - -Writes and removals may target middleware-visible end-to-end request headers. Protected credential, routing, framing, and hop-by-hop headers are always rejected. Header values must not contain control characters or OpenShell credential placeholder syntax. Middleware runs before credential injection, but it cannot introduce a value that the later injection step would resolve. - -OpenShell validates and applies each stage's mutations atomically. An invalid operation discards every mutation from that stage and follows its `on_error` behavior. Built-in failures can name the offending header. Operator-run failures use a platform-owned error code so request-derived header text cannot reach logs or denied responses. - -## Operate Middleware Services - -Plan startup and updates around these boundaries: - -- Start registered services before the gateway. The gateway validates every registration during startup. -- Keep service endpoints reachable from both the gateway and sandbox supervisors. The supervisors call operator-run services directly on the request path. -- Restart the gateway after changing registrations. -- Keep required services available before creating or updating policies. The gateway validates implementation-owned config before persisting a policy. -- Treat `fail_open` as an explicit availability-over-enforcement decision. - -When the effective sandbox configuration changes, a running supervisor validates the new service registry before installing it. If the reload fails, the supervisor keeps its last-known-good registry and emits a configuration failure event. - -## Observe Middleware - -Middleware activity is emitted through OpenShell's OCSF logging: - -- Each invocation records its policy-local config name, attached middleware name, decision, transformation state, and failure state. -- A denied invocation records a platform-owned reason derived from the policy-local config name and optional validated reason code. OpenShell does not record service-provided free-form reason text. -- A bypass under `fail_open` emits a detection finding. -- A required stage that fails closed emits a high-severity detection finding. -- A host-matched attachment without a WebSocket binding emits an informational `binding_not_selected` coverage event. -- A binary message encountered by an active WebSocket stage emits an informational `unsupported_message_type` coverage event with message type, sequence, and byte count. It is not reported as an invocation or failure. -- Built-in findings include their type, label, and aggregate count. Operator-run findings use the operator-owned registration name and a platform label plus the aggregate count; OpenShell does not log service-provided finding text or diagnostic metadata. A stage can return at most 32 findings. Exceeding the per-stage cap is an invalid response handled through `on_error`. A maximum 10-stage chain retains and emits up to 320 findings without silently dropping findings from later stages. -- Registry reload success and failure are emitted as configuration state changes. - -See [Logging](/observability/logging) for log access and [OCSF JSON Export](/observability/ocsf-json-export) for structured export. - -## Runnable example - -The [content guard example](https://github.com/NVIDIA/OpenShell/tree/main/examples/supervisor-middleware-content-guard) matches configured literal terms in UTF-8 request bodies, complete response bodies, and client WebSocket text messages. It supports redaction and denial. Responses require whole-body inspection; unavailable inspection or invalid UTF-8 invokes the configured `on_error` policy. It is not a general PII detector. - -The example includes a policy, local fixture, and smoke launcher. - -## Current Limitations - -- Middleware applies only through operation bindings advertised by each implementation. For protocols that have no supported middleware operation at all, such as HTTP/2 prior knowledge or non-HTTP TCP, the existing uninspectable-traffic gate denies a host match containing `fail_closed` and relays an all-`fail_open` match with a detection finding. -- The typed operation and phase pairs are `HTTP_REQUEST/PRE_CREDENTIALS`, `HTTP_RESPONSE/PRE_RETURN`, and `WEBSOCKET_MESSAGE/PRE_CREDENTIALS`. -- A host match does not imply every advertised operation: an HTTP-only attachment can inspect the upgrade GET, then post-upgrade traffic passes with `binding_not_selected` coverage. -- The V1 WebSocket binding inspects complete client text messages only. Binary messages pass with `unsupported_message_type` coverage for active stages; control frames and upstream-to-client messages remain outside the middleware operation. -- Selection uses destination host include and exclude patterns. -- A fail-closed middleware cannot cover `tls: skip` endpoints because OpenShell cannot inspect that traffic. An all-`fail_open` match may cover the endpoint; OpenShell bypasses the middleware and emits a detection finding. -- Operator-run services use TLS `https://` when gateway JWT signing is enabled, unless the registration sets `allow_insecure_transport`. Certificates must chain to the configured custom CA or platform roots, and the endpoint hostname must match. -- Extension tokens and sandbox-to-gateway tokens are signed by the same key. They are separated by audience and by `typ`, but the extension credential path cannot yet be rotated or revoked independently of sandbox admission. -- OpenShell does not track or revoke `jti`; bearer tokens can be replayed until expiry. Per-request replay resistance requires proof of possession or request binding. -- mTLS client authentication, health checks, runtime registration, and overlapping signing-key rotation are not available. diff --git a/docs/extensibility/supervisor-middleware/configure.mdx b/docs/extensibility/supervisor-middleware/configure.mdx new file mode 100644 index 0000000000..0a42dc1412 --- /dev/null +++ b/docs/extensibility/supervisor-middleware/configure.mdx @@ -0,0 +1,199 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Supervisor Middleware Configuration" +sidebar-title: "Configuration" +slug: "extensibility/supervisor-middleware/configure" +description: "Attach supervisor middleware to sandbox traffic, register your own middleware services, and observe middleware decisions." +keywords: "Supervisor Middleware, Configuration, Network Policy, Operations, OCSF, Logging" +--- + +Register your middleware service in gateway configuration, then attach it to destination hosts in sandbox policy. This page also covers ordering, failure behavior, service lifecycle, and observability. Built-in middleware needs only a policy entry. + +## Use Your Own Middleware Service + +A middleware service is a gRPC server that implements the supervisor middleware protocol. The [content guard example](https://github.com/NVIDIA/OpenShell/tree/main/examples/supervisor-middleware-content-guard) implements request, response, and WebSocket checks in one service, and includes a policy and a local launcher. + +To use your own service: + +1. Start the service where both the gateway and sandbox supervisors can reach it. +2. Register it in gateway configuration. +3. Restart the gateway. +4. Attach it in sandbox policy by setting `middleware` to the registration `name`. + +Register the service in gateway TOML: + +```toml +[[openshell.supervisor.middleware]] +name = "content-guard" +grpc_endpoint = "https://content-guard.example:50051" +tls_ca_cert_path = "/path/to/content-guard-ca.pem" +max_payload_bytes = 262144 +timeout = "500ms" +``` + + + Unique name that policies use to attach the service. The `openshell/` prefix is reserved for built-ins. + + + + Service address reachable from the gateway and sandbox supervisors. + + + + Largest request body, response body, or WebSocket message the service inspects. Must be at most 4 MiB and no larger than the service advertises. + + + + Time limit for `Describe`, `ValidateConfig`, and individual evaluations or stream exchanges. Use an integer followed by `ms` or `s`, from `10ms` through `30s`. A binding may advertise a shorter evaluation timeout. Accepted HTTP response and WebSocket streams have no connection-wide deadline. + + + + Private CA for the service certificate. Without it, OpenShell uses platform trust roots. + + + + Token audience. Refer to [Extension Authentication](/extensibility/extension-authentication). + + + + Allows a plaintext `http://` endpoint with no authentication. Use it for local development or on a network that already authenticates callers. + + +At startup, the gateway contacts every registered service to read its capabilities and verify [protocol compatibility](/extensibility/extension-negotiation). The gateway does not start if a service is unavailable or incompatible. [Gateway Configuration](/reference/gateway-config#supervisor-middleware-services) describes the full TOML context. + +When the gateway has JWT signing configured, OpenShell authenticates every call to your service with a short-lived token. [Extension Authentication](/extensibility/extension-authentication) describes how your service validates it. + +## Attach Middleware in Policy + +Add entries to the top-level `network_middlewares` map in sandbox policy. Each entry selects destination hosts and names the middleware to run for them: + + + + +Attach a [registered service](#use-your-own-middleware-service) by its registration name. Your service defines and validates the `config` object: + +```yaml showLineNumbers={false} +network_middlewares: + content-guard: + middleware: content-guard + config: + mode: redact + terms: ["internal-only"] + replacement: "[FILTERED]" + endpoints: + include: ["api.example.com"] +``` + + + + +Run two middleware for the same hosts. `regex-demo` runs before `content-guard` because it has the lower `order`, and neither runs for `trusted.example.com`: + +```yaml showLineNumbers={false} +network_middlewares: + regex-demo: + middleware: openshell/regex + order: 10 + endpoints: + include: ["*.example.com"] + exclude: ["trusted.example.com"] + content-guard: + name: Content guard + middleware: content-guard + order: 20 + endpoints: + include: ["*.example.com"] + exclude: ["trusted.example.com"] +``` + + + + +Attach the built-in `openshell/regex` middleware to one host: + +```yaml showLineNumbers={false} +network_middlewares: + regex-demo: + middleware: openshell/regex + endpoints: + include: ["api.example.com"] +``` + + + +`openshell/regex` is a regex redactor used for demonstration only. It lets you try middleware without running your own service. It replaces text that matches a fixed `sk-` pattern in UTF-8 HTTP bodies and outgoing WebSocket text messages, and the pattern is not configurable. Do not rely on it to detect or remove sensitive data. + + + + + + +### Entry Fields + +The map key, such as `content-guard`, is the entry's stable identity in logs. Each entry accepts these fields: + + + A built-in middleware name, such as `openshell/regex`, or the `name` of a registered service. + + + + Destination hosts to run the middleware for. Uses the same host patterns as network policy endpoints. + + + + Destination hosts to skip. Takes precedence over `include`. + + + + Position in the chain when several entries match the same host. Lower values run first. Values must be unique within a policy, so set them explicitly when a policy has more than one entry. + + + + What happens when the middleware fails. Refer to [Choose Failure Behavior](#choose-failure-behavior). + + + + Configuration passed to the middleware. The middleware defines and validates its format. + + + + Display name. Defaults to the map key. + + +[Policy Schema](/reference/policy-schema#network-middleware) lists every field and limit. + +## Choose Failure Behavior + +Middleware fails when it cannot complete its check, for example when its service is unavailable, times out, returns an invalid result, or receives a payload over its limit. By default, OpenShell blocks the affected request, response, or WebSocket connection. The `on_error` field on a policy entry controls this behavior. + + + +`fail_open` skips failed middleware and lets the traffic continue. Avoid it unless availability matters more than enforcement for that traffic. Future releases may remove it. OpenShell emits a detection finding each time it skips middleware. + + + +A denial from middleware always blocks traffic, regardless of `on_error`. Middleware decisions also stay enforced when the endpoint uses `enforcement: audit`. To observe traffic without blocking it, have your middleware allow the traffic and report findings. + +`on_error` does not extend what middleware can inspect. The [current limitations](/extensibility/supervisor-middleware#current-limitations) apply under both settings. With the default behavior, an entry cannot select a `tls: skip` endpoint, because OpenShell cannot inspect that traffic. + +## Operate Middleware Services + +Middleware services sit on the request path of every sandbox that uses them. Plan their availability accordingly: + +- Start registered services before the gateway, and restart the gateway after changing registrations. +- Keep services available when creating or updating policies. The gateway asks the service to validate its `config` before it accepts a policy. +- Keep services reachable from sandbox supervisors, which call them directly for each request. + +When a running sandbox receives a configuration change, its supervisor validates the new middleware setup before using it. If validation fails, the supervisor keeps its previous setup and emits a configuration failure event. + +## Observe Middleware Decisions + +OpenShell records middleware activity in its [OCSF logs](/observability/logging): + +- Each invocation records the policy entry, middleware name, decision, whether it changed the traffic, and any failure. +- Failed middleware emits a detection finding, whether OpenShell blocks the traffic or skips the middleware. +- Coverage events record traffic that selected middleware could not inspect, such as WebSocket binary messages. +- Configuration changes emit state-change events on success and failure. + +For denials and findings, OpenShell logs the middleware name and a validated reason code or finding label. It does not log free-form text from your service. [OCSF JSON Export](/observability/ocsf-json-export) describes how to send these events to other systems. diff --git a/docs/extensibility/supervisor-middleware/index.mdx b/docs/extensibility/supervisor-middleware/index.mdx new file mode 100644 index 0000000000..50c5835621 --- /dev/null +++ b/docs/extensibility/supervisor-middleware/index.mdx @@ -0,0 +1,111 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Supervisor Middleware" +slug: "extensibility/supervisor-middleware" +description: "Inspect, block, or change sandbox requests and responses with supervisor middleware." +keywords: "Generative AI, Cybersecurity, AI Agents, Supervisor Middleware, Extensibility, Request Filtering" +--- + +Supervisor middleware lets you inspect, block, or change the content that an agent sends and receives over the network. The sandbox supervisor, which enforces the sandbox's network policy, runs middleware as traffic passes between the agent and an external service. + +## When to Use Middleware + +A network policy decides whether an agent can call an API. Use middleware when you also need to check what the agent sends to that API or what the API returns, without changing the agent's code. For example, you can: + +- Redact API tokens from outgoing request bodies. +- Reject requests that contain content your organization prohibits. +- Remove sensitive content from HTTP responses before the agent receives them. +- Check outgoing WebSocket text messages during a session. + +The [content guard example](https://github.com/NVIDIA/OpenShell/tree/main/examples/supervisor-middleware-content-guard) shows a complete middleware service that performs checks like these. + +## How Middleware Processes Traffic + +The sandbox supervisor calls middleware after it checks network policy and before it forwards traffic to the external service: + +```mermaid +sequenceDiagram + participant Sandbox + participant Supervisor + participant Middleware as Middleware service + participant Service as External service + + Sandbox->>Supervisor: Request + Supervisor->>Supervisor: Check network policy + Supervisor->>Middleware: Check request + Middleware-->>Supervisor: Allow, deny, or change + Supervisor->>Supervisor: Inject provider credentials + Supervisor->>Service: Request + Service-->>Supervisor: Response + Supervisor->>Middleware: Check response + Middleware-->>Supervisor: Allow, deny, or change + Supervisor-->>Sandbox: Response +``` + +OpenShell checks network policy first, so middleware only sees traffic that policy allows. It then selects middleware by destination host and runs it in the order you configure. OpenShell injects provider credentials after request middleware runs, so middleware never sees them. + +Each middleware decides whether to allow, deny, or change the traffic. A denial blocks the request even when network policy allows it. If middleware fails or its service is unavailable, OpenShell blocks the affected traffic by default. + +Middleware can handle three operations: HTTP requests, HTTP responses, and the text messages an agent sends over a WebSocket connection. Each middleware declares which operations it supports. [Supported Operations](/extensibility/supervisor-middleware/operations) describes each one. + +## Current Limitations + +- Middleware does not inspect WebSocket binary messages or messages that the external service sends to the agent. +- Response middleware cannot inspect the body of compressed, partial, or `Cache-Control: no-transform` responses. +- Middleware cannot inspect traffic to endpoints that use `tls: skip`. + +If your deployment requires inspection of this traffic, middleware cannot enforce that requirement today. + +## Configure Middleware + +To add middleware to your sandboxes, start with configuration: + + + + + +Attach built-in middleware, register your own service, choose failure behavior, and observe middleware decisions. + + + + +## Build Your Middleware Service + +A middleware service is a gRPC server that implements the services in [`proto/supervisor_middleware.proto`](https://github.com/NVIDIA/OpenShell/blob/main/proto/supervisor_middleware.proto). The [content guard example](https://github.com/NVIDIA/OpenShell/tree/main/examples/supervisor-middleware-content-guard) is a complete service you can start from. + + + +The middleware API is still evolving. Future versions will change it to make the contract consistent across HTTP requests, HTTP responses, and WebSocket messages. Expect to update your service when you upgrade OpenShell. [Protocol negotiation](/extensibility/extension-negotiation) detects incompatible versions at startup. + + + +Every middleware service implements these RPCs: + +- `Describe` returns the service manifest. The manifest lists the operations the service supports, with a payload limit and optional timeout for each. It also carries [protocol negotiation](/extensibility/extension-negotiation) metadata and the [expected audience](/extensibility/extension-authentication#confirm-the-audience-at-startup). +- `ValidateConfig` checks the `config` object from a policy entry. The gateway calls it before it accepts a policy. + +The service then implements an evaluation RPC for each operation it supports, as described in [Supported Operations](/extensibility/supervisor-middleware/operations). + +Most gRPC servers reject incoming messages larger than 4 MiB by default. Raise your server's limit to at least 4.5 MiB so that a maximum-size payload and the rest of the message fit. + +These guides cover the rest of the service contract: + + + + + +When OpenShell calls your service, what it receives, and what it can return for HTTP requests, HTTP responses, and WebSocket messages. + + + + +How to verify that calls to your service come from your OpenShell gateway. + + + + +How your service and OpenShell agree on a protocol version at startup. + + + diff --git a/docs/extensibility/supervisor-middleware/operations.mdx b/docs/extensibility/supervisor-middleware/operations.mdx new file mode 100644 index 0000000000..50a01b33f9 --- /dev/null +++ b/docs/extensibility/supervisor-middleware/operations.mdx @@ -0,0 +1,251 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Supported Operations" +sidebar-title: "Supported Operations" +slug: "extensibility/supervisor-middleware/operations" +description: "Build a middleware service that inspects and changes sandbox HTTP requests, HTTP responses, and WebSocket messages." +keywords: "Supervisor Middleware, Operations, Bindings, HTTP, WebSocket, Request Body, Response Body, Header Mutation, Payload Limits" +--- + +A middleware service handles one or more operations, such as HTTP requests or WebSocket messages. This page describes when OpenShell calls your service for each operation, what your service receives, and what it can return. The RPCs that every middleware service implements are described in [Build Your Middleware Service](/extensibility/supervisor-middleware#build-your-middleware-service). + +## Operations and Phases + +| Operation | Phase | When OpenShell calls your service | RPC | +| --- | --- | --- | --- | +| `HTTP_REQUEST` | `PRE_CREDENTIALS` | After network policy allows a request, before OpenShell injects provider credentials. | `EvaluateHttpRequest` | +| `HTTP_RESPONSE` | `PRE_RETURN` | After the external service responds, before the sandbox receives the response. | `HttpResponsePreReturn.Evaluate` | +| `WEBSOCKET_MESSAGE` | `PRE_CREDENTIALS` | When the sandbox opens a WebSocket connection, and for each text message it sends. | `EvaluateWebSocketSession` | + +Your service declares each operation it supports as a binding, an operation and its phase, in its `Describe` manifest. OpenShell calls your service only for the operations it declares. + +When several middleware match the same traffic, OpenShell calls them in ascending `order`, and each middleware sees the changes that earlier middleware made. + +## Common Inputs and Results + +Every operation includes a request context. The context always includes `sandbox_id`. Use it for authorization, persistence, and correlation. The context also includes `sandbox` and `workspace` names when available. Names can be reused, so use them only for display and logging. + +Operations that include headers deliver them in wire order, and repeated headers stay separate. OpenShell removes credential, routing, and hop-by-hop headers before sending them to your service. It also removes framing headers from request input. Response preflight retains upstream `Content-Length`, `Content-Encoding`, and `Content-Range` as read-only metadata, unless `Connection` names them as hop-by-hop headers. Your service cannot change or remove these fields; OpenShell handles downstream framing. + +Every result can include an optional reason code, findings, and metadata: + +- A reason code identifies why your service denied traffic. OpenShell includes it in logs and in HTTP error responses. It must be 1-64 bytes long, start with a lowercase ASCII letter, and contain only lowercase ASCII letters, digits, and underscores. OpenShell treats an invalid code as a middleware failure. +- Findings report what your service detected, even when it allows the traffic. OpenShell logs how many findings your service reported, not their text. +- Metadata carries non-secret diagnostic details. OpenShell does not log it. + +OpenShell never forwards free-form text from your service to the sandbox or to logs. + +## HTTP + +HTTP middleware can check a request before it leaves the sandbox, and the response before the sandbox receives it. A service can support either operation or both. + +### Requests + +OpenShell calls your service after network policy allows a request and before it injects provider credentials. Your service can allow, deny, or change the request. + +```mermaid +sequenceDiagram + participant Sandbox + participant Supervisor + participant Middleware + participant Service as External service + + Sandbox->>Supervisor: HTTP request + Supervisor->>Supervisor: Check network policy + Supervisor->>Middleware: EvaluateHttpRequest + Middleware-->>Supervisor: Allow, deny, or change + Supervisor->>Supervisor: Inject provider credentials + Supervisor->>Service: Forward request +``` + + + + +The request body, target, headers, and request context. + + + + +- Allow the request unchanged. +- Deny the request with an optional reason code. +- Replace the body, up to the same payload limit as the input. +- Add, change, or remove headers. + +When your service replaces the body, OpenShell checks the new body against body-aware policy, such as GraphQL, JSON-RPC, or MCP rules, before the next middleware or the external service sees it. A replacement cannot introduce an operation that policy denies. + + + + +When your service denies a request, the sandbox receives a structured error and OpenShell does not contact the external service: + +```json +{ + "error": "middleware_denied", + "detail": "Request rejected by configured middleware", + "policy": "api-policy", + "middleware": "content-guard", + "reason_code": "content_match" +} +``` + +When middleware fails, the sandbox receives `error: middleware_failed` instead. + + + + +A result can include an ordered list of header changes. A `write` change sets how it treats an existing header with the same name: + +- `append` adds another value. +- `overwrite` replaces all existing values. +- `skip` keeps the existing values. + +A `remove` change removes all values for the name. OpenShell protects credential, routing, framing, and connection headers, and for responses also status, authentication challenge, content coding, range, and security policy headers. Header values cannot contain control characters, and request header values cannot contain OpenShell credential placeholders. + +OpenShell applies each middleware's changes as a unit. If one change is invalid, it discards all of them and treats the result as a middleware failure. + + + + +### Responses + +OpenShell calls your service after the external service responds and before the sandbox receives the response. At preflight, your service can skip inspection, block delivery, or choose how to inspect the response. During inspection, it can change or block the response. + +```mermaid +sequenceDiagram + participant Sandbox + participant Supervisor + participant Middleware + participant Service as External service + + Service-->>Supervisor: HTTP response + Supervisor->>Middleware: Preflight with status and headers + Middleware-->>Supervisor: Skip, inspect, or block + Supervisor->>Middleware: Body, when inspecting + Middleware-->>Supervisor: Pass, change, or stop + Supervisor-->>Sandbox: Deliver response +``` + + + + +The preflight includes the status and headers. A preflight result can skip inspection, block delivery with `block_delivery`, or inspect the response with one body mode: + +| Mode | Behavior | +| --- | --- | +| `HEADERS_ONLY` | Inspects the status and headers without the body. | +| `WHOLE_BODY_BYTES` | Receives the complete body in one piece before OpenShell sends anything to the sandbox. | +| `STREAM_BYTES` | Receives the body in ordered pieces of at most 64 KiB, so delivery can start before the body ends. | + +Body inspection is unavailable for responses without a body, partial responses, compressed responses, and responses with `Cache-Control: no-transform`. For those, a middleware that requests body inspection fails. Interim `1xx` responses pass through unchanged, and a `101` protocol upgrade skips response middleware. + + + + +- Pass the response or body piece unchanged. +- Replace the body or a body piece. +- Add, change, or remove headers, with the same rules as requests. +- Block delivery with an optional reason code. + + + + +If your service blocks the response before OpenShell starts sending it, the sandbox receives `403 Forbidden` with the same `middleware_denied` error as a denied request. If middleware fails before then, the sandbox receives `502 Bad Gateway` with `error: response_delivery_failed`. If delivery has already started, OpenShell aborts the response in both cases. + +The external service has already handled the request, so blocking its response does not undo it, and retrying the request may repeat side effects. + + + + +A service that inspects the body also receives trailers after the body. A preflight can declare new trailer names, and the service can then add them. + +After middleware changes the body, OpenShell recalculates `Content-Length` and framing, so your service never handles transfer encoding. + + + + +## WebSocket + +WebSocket middleware checks the text messages the sandbox sends over a WebSocket connection. + +### Messages + +OpenShell calls your service when the sandbox opens a WebSocket connection, then for each text message the sandbox sends. Your service can allow, deny, or replace each message. + +```mermaid +sequenceDiagram + participant Sandbox + participant Supervisor + participant Middleware + participant Service as External service + + Sandbox->>Supervisor: WebSocket upgrade request + Supervisor->>Supervisor: Check network policy + Supervisor->>Middleware: Preflight + Middleware-->>Supervisor: Inspect + Supervisor->>Service: Forward upgrade + Service-->>Supervisor: Connection accepted + Supervisor->>Middleware: Session start + Sandbox->>Supervisor: Text message + Supervisor->>Middleware: Complete text message + Middleware-->>Supervisor: Allow or replace + Supervisor->>Service: Forward message + Supervisor->>Middleware: Session end +``` + + + + +OpenShell opens one bidirectional stream to your service for each connection and sends these events: + +1. A preflight before contacting the external service. Your service returns `INSPECT`, `SKIP`, or `DENY`. `SKIP` closes the middleware stream and passes all messages without inspection by that middleware. `DENY` rejects the upgrade. +2. A session start after the external service accepts the connection. It includes the negotiated subprotocol. +3. Each complete text message from the sandbox, in order. +4. A session end when the connection closes, sent on a best-effort basis. + +Your service returns a result for each preflight and message. Session start and end are notifications that need no result. When OpenShell closes its side of the stream, finish your side of the stream too. + + + + +OpenShell reassembles fragmented messages and decompresses `permessage-deflate` messages before sending them, so your service always receives a complete UTF-8 text message. A replacement can be any text, including an empty string. OpenShell re-frames and, when needed, re-compresses the replacement before forwarding it. + +OpenShell does not send binary messages, control frames, or messages from the external service to middleware. Binary messages pass through unchanged, and OpenShell records an `unsupported_message_type` coverage event. Message sequence numbers count binary messages too, so your service can see gaps between text messages. + +Only middleware that declares `WEBSOCKET_MESSAGE` receives messages. Middleware that declares only `HTTP_REQUEST` still inspects the upgrade request, and OpenShell records a `binding_not_selected` coverage event for it. + + + + +A denied message closes the connection with code `1008`. When middleware fails, OpenShell rejects the upgrade or closes the connection. With `fail_open`, OpenShell instead stops using that middleware for the rest of the connection. + + + + +| Code | Meaning | +| --- | --- | +| `1002` | WebSocket protocol error. | +| `1007` | Invalid UTF-8 text. | +| `1008` | Middleware or policy denial. | +| `1009` | Text message exceeds the 4 MiB platform limit. | +| `1012` | A policy change requires the connection to restart. | +| `1013` | Message assembly capacity is full. | + + + + +## Limits + +| Limit | Applies to | Value | +| --- | --- | --- | +| Payload per middleware | All operations | The registration's `max_payload_bytes`, up to 4 MiB. Applies to a request body, a whole response body or one streamed piece, or one text message, and to each replacement. | +| Time per evaluation or stream exchange | All operations | The smaller of the registration `timeout` and the binding timeout in the service manifest. Accepted HTTP response and WebSocket streams have no connection-wide deadline. | +| Streamed body piece | HTTP responses | 64 KiB. | +| Time for all middleware on one streamed piece | HTTP responses | 30 seconds. | +| Fragments per message | WebSocket messages | 4,096. | +| Time to assemble a message | WebSocket messages | 30 seconds without progress, or 2 minutes in total. | +| Time to forward a message | WebSocket messages | 2 minutes. | + +When a payload exceeds one middleware's limit, that middleware fails. Other middleware keep their own limits. + +Middleware evaluation capacity in each sandbox is shared by all operations. When it is full, OpenShell returns `503 Service Unavailable` for an HTTP request before reading its body. When WebSocket message assembly capacity is full, OpenShell closes the connection with code `1013`. diff --git a/docs/index.yml b/docs/index.yml index 30241862fb..d549ac1986 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -72,10 +72,20 @@ navigation: slug: extensibility contents: - page: "Overview" + path: extensibility/overview.mdx + - page: "Protocol Negotiation" path: extensibility/extension-negotiation.mdx - - page: "Middleware" - path: extensibility/supervisor-middleware.mdx - - page: "Interceptors" + - page: "Authentication" + path: extensibility/extension-authentication.mdx + - section: "Supervisor Middleware" + slug: supervisor-middleware + path: extensibility/supervisor-middleware/index.mdx + contents: + - page: "Configuration" + path: extensibility/supervisor-middleware/configure.mdx + - page: "Supported Operations" + path: extensibility/supervisor-middleware/operations.mdx + - page: "Gateway Interceptors" path: extensibility/gateway-interceptors.mdx - page: "Drivers" path: extensibility/drivers.mdx diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index e032ff8a89..d9b3e489c0 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -372,7 +372,7 @@ The service `grpc_endpoint` supports plaintext `http://` and TLS `https://`. HTT When `gateway_jwt` is configured, OpenShell attaches short-lived bearer credentials to gateway and supervisor calls and requires `https://`. A middleware endpoint must be reachable from sandbox supervisors, so Unix sockets are not an option here. Set `allow_insecure_transport = true` on a registration to keep a plaintext `http://` endpoint: OpenShell then attaches no credential, supervisors do not request one, and the gateway logs a warning naming the registration at every startup. mTLS client authentication, health checks, and runtime registration are not currently supported. The endpoint must be reachable from both the gateway and sandbox supervisors; use `host.openshell.internal` or another shared address that can be resolved in both places. -See [Supervisor Middleware](/extensibility/supervisor-middleware) for selection, failure, payload-limit, and operational guidance. +See [Supervisor Middleware Configuration](/extensibility/supervisor-middleware/configure) for attachment, failure, and operational guidance. ## Gateway Interceptors diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index b0c84cfa0c..f3072da5b5 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -633,7 +633,7 @@ network_middlewares: Host selectors use the same case-insensitive exact and DNS glob semantics as network endpoints: `*` matches exactly one DNS label and `**` matches one or more labels, so `**.example.com` covers subdomains but not `example.com` itself. Brace alternates are rejected at validation. A matching attachment joins only the operation chains its implementation advertises. An HTTP-only attachment may inspect a WebSocket upgrade GET without joining the post-upgrade chain; OpenShell permits the messages and records `binding_not_selected` coverage regardless of `on_error`. WebSocket bindings inspect complete client text messages. Binary messages pass with `unsupported_message_type` coverage for active stages. A fail-closed selector that can cover a `tls: skip` endpoint is rejected because OpenShell cannot inspect that traffic through any operation. An all-`fail_open` match may cover the endpoint; the supervisor bypasses the middleware and emits a detection finding. -See [Supervisor Middleware](/extensibility/supervisor-middleware) for registration, failure behavior, body limits, and operational guidance. +See [Supervisor Middleware Configuration](/extensibility/supervisor-middleware/configure) for registration, failure behavior, and operational guidance. ## Full Example diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 698701d505..37eb13abb1 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -120,7 +120,7 @@ Matching entries run once each by ascending `order`; lower values run first, and `on_error` defaults to `fail_closed`. Use `fail_open` only when skipping a selected stage that fails is acceptable. For WebSocket streams, a broken fail-open stage is disabled for the rest of that connection and OpenShell emits a state-change finding. A host-matched attachment joins only operation chains advertised by its implementation. An HTTP-only attachment may inspect the WebSocket upgrade GET, but post-upgrade messages pass with informational `binding_not_selected` coverage under either error mode. Binary messages also pass without middleware inspection and produce `unsupported_message_type` coverage for active WebSocket stages. Upstream-to-client messages remain uninspected. Policy validation rejects a fail-closed selector that can cover a `tls: skip` endpoint. An all-`fail_open` match may cover the endpoint; the supervisor bypasses the middleware and emits a detection finding. -See [Supervisor Middleware](/extensibility/supervisor-middleware) for registration, chain ordering, body limits, failure behavior, and operations. +See [Supervisor Middleware](/extensibility/supervisor-middleware) for an introduction, or [Supervisor Middleware Configuration](/extensibility/supervisor-middleware/configure) for registration, ordering, failure behavior, and operations. ## Baseline Filesystem Paths From 08548713c56856038765bd84ae6ae64b263fee19 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 24 Sep 2026 17:57:42 -0700 Subject: [PATCH 5/7] fix(install): honor pinned releases and speed up prerelease discovery (#3681) * fix(install): use native packages for prereleases and speed up discovery Signed-off-by: Drew Newberry * fix(install): honor pinned versions and guard Snap migration Signed-off-by: Drew Newberry * fix(install): omit Snap migration guard Signed-off-by: Drew Newberry --------- Signed-off-by: Drew Newberry --- README.md | 3 +- architecture/build.md | 6 ++ docs/about/installation.mdx | 18 +++-- install.sh | 124 ++++++++++++++----------------- tasks/scripts/test-install-sh.sh | 98 ++++++++++++++---------- 5 files changed, 131 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 4972acad1d..fd9b894ec7 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,8 @@ curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | OPENSHELL_VERSION=pre sh ``` -The installer downloads only the artifact for the current platform and rejects expired candidates during discovery. Installed packages retain the candidate's exact version, such as `0.1.0-pre.3`. Prerelease tags do not create entries on the GitHub Releases page. +The installer checks prerelease tags from newest to oldest, selects an unexpired artifact from a successful release run for the current platform, and downloads only that artifact. Installed packages retain the candidate's exact version, such as `0.1.0-pre.3`. Prerelease tags do not create entries on the GitHub Releases page. +On Linux, prereleases and explicit release tags use Debian or RPM packages even if `snap` is installed. The rolling [`dev` release](https://github.com/NVIDIA/OpenShell/releases/tag/dev) does not require GitHub authentication: diff --git a/architecture/build.md b/architecture/build.md index 87cd8deb24..6b5077d966 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -324,6 +324,12 @@ positive canary uses system Docker; negative preflight coverage verifies that the installer rejects both missing Docker and the Docker Snap before installing OpenShell. Its Debian lane removes snapd before running the installer so Snap precedence cannot change the package under test. +Explicit release tags and the `pre` alias bypass Snap selection and use the +native Debian or RPM package path even when `snap` is available. The `pre` alias +checks matching Git tags in version order, then looks up the exact platform +artifact and verifies the release run instead of listing every repository +artifact. + Snapd runs the gateway as a root-owned system service. Its generated client certificates reside in root-owned snap state and are unavailable to ordinary CLI users, so the Snap uses plaintext loopback transport and enables unauthenticated diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 45206d5772..8a8ac97689 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -19,11 +19,11 @@ Install OpenShell with a single command: curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -The script detects your operating system and installs the OpenShell CLI, standalone policy prover, and gateway. On Linux, it installs from the Snap Store whenever the `snap` command is available. Otherwise, it uses the native Debian or RPM package. It then starts the local gateway server so you can begin creating sandboxes. +The script detects your operating system and installs the OpenShell CLI, standalone policy prover, and gateway. On Linux, it installs from the Snap Store when `snap` is available and `OPENSHELL_VERSION` is unset or `dev`. Explicit release tags and prereleases use the native Debian or RPM package. It then starts the local gateway server so you can begin creating sandboxes. Snap installs use `latest/stable` by default and `latest/edge` when -`OPENSHELL_VERSION=dev`. Other `OPENSHELL_VERSION` values also use the stable -Snap channel because Snap Store channels do not select an exact GitHub release. +`OPENSHELL_VERSION=dev`. Set `OPENSHELL_VERSION` to a release tag to install +that exact Debian or RPM version, even if `snap` is available. You can also download release artifacts directly from the [OpenShell GitHub Releases](https://github.com/NVIDIA/OpenShell/releases) page. @@ -75,13 +75,15 @@ brew services restart openshell ## Linux On Linux systems with the `snap` command, the install script uses the OpenShell -snap. Install and start Docker Engine from a system package or Docker's package -repository before running the installer. The Docker snap is not currently -compatible with OpenShell. +snap when `OPENSHELL_VERSION` is unset or `dev`. Explicit release tags and the +`pre` prerelease alias use Debian or RPM packages even when `snap` is available. +For Snap installs, install and start Docker Engine from a system package or +Docker's package repository first. The Docker snap is not currently compatible +with OpenShell. -On Fedora and RHEL without `snap`, the install script uses RPM packages. The RPMs install the `openshell` CLI, `openshell-prover`, the `openshell-gateway` daemon, and a systemd user service. +On Fedora and RHEL without `snap`, or when a release tag or `pre` is specified, the install script uses RPM packages. The RPMs install the `openshell` CLI, `openshell-prover`, the `openshell-gateway` daemon, and a systemd user service. -On Debian and Ubuntu without `snap`, the install script uses a Debian package. The Debian package installs the `openshell` CLI, `openshell-prover`, the `openshell-gateway` daemon, VM sandbox support, and a systemd user service. +On Debian and Ubuntu without `snap`, or when a release tag or `pre` is specified, the install script uses a Debian package. The Debian package installs the `openshell` CLI, `openshell-prover`, the `openshell-gateway` daemon, VM sandbox support, and a systemd user service. Linux packages require glibc 2.28 or newer. The installer checks libc before downloading packages and exits with an error on older glibc versions, Alpine, musl-based distributions, or unknown libc environments. diff --git a/install.sh b/install.sh index 8b8eee90c0..60bfd6f64c 100755 --- a/install.sh +++ b/install.sh @@ -66,14 +66,16 @@ NOTES: When OPENSHELL_VERSION is unset, this resolves the latest tagged release from ${GITHUB_URL}/releases/latest. - On Linux, the installer uses the OpenShell snap whenever the snap command - is available. It installs from latest/edge when OPENSHELL_VERSION=dev and - from latest/stable otherwise. The OpenShell snap requires a running Docker - Engine installed from a system package or Docker's package repository. The - Docker snap is not currently compatible with OpenShell. - - Without snap, Linux installs the Debian package on amd64/arm64 or the RPM - packages on x86_64/aarch64, depending on the host package manager. + On Linux, the installer uses the OpenShell snap when the snap command is + available and OPENSHELL_VERSION is unset or dev. Snap installs use + latest/stable by default and latest/edge for dev. Explicit release tags + and prereleases use Debian or RPM packages. The OpenShell snap requires a + running Docker Engine installed from a system package or Docker's package + repository. The Docker snap is not currently compatible with OpenShell. + + For explicit versions or without snap, Linux installs the Debian package + on amd64/arm64 or the RPM packages on x86_64/aarch64, depending on the + host package manager. macOS installs the release Homebrew formula on Apple Silicon and starts a brew services-backed local gateway. EOF @@ -432,61 +434,39 @@ resolve_latest_prerelease_tag() { info "resolving latest prerelease..." _artifact_platform="$(prerelease_artifact_platform)" - _successful_run_ids="$(gh api --paginate \ - "repos/${REPO}/actions/workflows/release-tag.yml/runs?status=success&per_page=100" \ - --jq '.workflow_runs[] | select(.status == "completed" and .conclusion == "success") | .id')" || { - error "failed to list successful Release Tag workflow runs" + _release_tags="$(gh api "repos/${REPO}/git/matching-refs/tags/v" --jq ' + [.[].ref | sub("^refs/tags/"; "") | + select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+-pre\\.[1-9][0-9]*$"))] | + sort_by(split("-pre.") as $parts | + ($parts[0] | ltrimstr("v") | split(".") | map(tonumber)) + + [($parts[1] | tonumber)]) | reverse | .[]')" || { + error "failed to list prerelease tags" } - _artifact_records="$(gh api --paginate \ - "repos/${REPO}/actions/artifacts?per_page=100" \ - --jq '.artifacts[] | select(.expired == false) | [.workflow_run.id, .name] | @tsv')" || { - error "failed to list prerelease artifacts" - } - _artifact_names="$(printf '%s\n--ARTIFACTS--\n%s\n' "$_successful_run_ids" "$_artifact_records" | awk -F '\t' ' - $0 == "--ARTIFACTS--" { - reading_artifacts = 1 - next - } - !reading_artifacts { - if ($1 ~ /^[0-9]+$/) successful_runs[$1] = 1 - next - } - $1 in successful_runs { - print $2 - } - ')" - _release_tags="$(printf '%s\n' "$_artifact_names" | sed -n "s/^openshell-\(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*-pre\.[1-9][0-9]*\)-${_artifact_platform}$/\1/p" | sort -u)" - - _latest_prerelease="$(printf '%s\n' "$_release_tags" | awk ' - /^v[0-9]+\.[0-9]+\.[0-9]+-pre\.[1-9][0-9]*$/ { - tag = $0 - sub(/^v/, "", tag) - split(tag, version_parts, "-pre[.]") - split(version_parts[1], core, "[.]") - sequence = version_parts[2] + 0 - - if (!found || core[1] + 0 > major || - (core[1] + 0 == major && core[2] + 0 > minor) || - (core[1] + 0 == major && core[2] + 0 == minor && core[3] + 0 > patch) || - (core[1] + 0 == major && core[2] + 0 == minor && core[3] + 0 == patch && sequence > prerelease)) { - selected = $0 - major = core[1] + 0 - minor = core[2] + 0 - patch = core[3] + 0 - prerelease = sequence - found = 1 - } - } - END { - if (found) print selected + + for _tag in $_release_tags; do + _artifact_name="openshell-${_tag}-${_artifact_platform}" + info "checking ${_tag} for ${_artifact_platform}..." + _run_ids="$(gh api \ + "repos/${REPO}/actions/artifacts?name=${_artifact_name}&per_page=100" \ + --jq '[.artifacts[] | select(.expired == false)] | + sort_by(.created_at) | reverse | .[] | .workflow_run.id')" || { + error "failed to find prerelease artifact ${_artifact_name}" } - ')" - if [ -z "$_latest_prerelease" ]; then - error "no unexpired prerelease artifacts found" - fi + for _run_id in $_run_ids; do + _successful="$(gh api "repos/${REPO}/actions/runs/${_run_id}" --jq ' + .status == "completed" and .conclusion == "success" and + (.path | startswith(".github/workflows/release-tag.yml"))')" || { + error "failed to check prerelease workflow run ${_run_id}" + } + if [ "$_successful" = "true" ]; then + printf '%s\n' "$_tag" + return 0 + fi + done + done - printf '%s\n' "$_latest_prerelease" + error "no unexpired prerelease artifacts found" } is_prerelease_tag() { @@ -678,9 +658,16 @@ local_gateway_endpoint() { } linux_package_method() { - if has_cmd snap; then - echo "snap" - elif has_cmd dpkg; then + case "${OPENSHELL_VERSION:-}" in + '' | dev) + if has_cmd snap; then + echo "snap" + return 0 + fi + ;; + esac + + if has_cmd dpkg; then echo "deb" elif has_cmd rpm; then echo "rpm" @@ -1226,14 +1213,11 @@ install_linux_rpm() { } openshell_snap_channel() { - if [ "${OPENSHELL_VERSION:-}" = "dev" ]; then - printf '%s\n' "latest/edge" - else - if [ -n "${OPENSHELL_VERSION:-}" ]; then - warn "OPENSHELL_VERSION=${OPENSHELL_VERSION} is ignored for Snap installs; using latest/stable" - fi - printf '%s\n' "latest/stable" - fi + case "${OPENSHELL_VERSION:-}" in + dev) printf '%s\n' "latest/edge" ;; + '') printf '%s\n' "latest/stable" ;; + *) error "Snap installs do not support OPENSHELL_VERSION=${OPENSHELL_VERSION}; use a native package" ;; + esac } ensure_snap_gateway_config() { diff --git a/tasks/scripts/test-install-sh.sh b/tasks/scripts/test-install-sh.sh index 9508f1201e..8cc6102148 100755 --- a/tasks/scripts/test-install-sh.sh +++ b/tasks/scripts/test-install-sh.sh @@ -102,13 +102,15 @@ assert_glibc_preflight_fails \ assert_linux_package_method() { local name=$1 - local snap_present=$2 - local dpkg_present=$3 - local rpm_present=$4 - local expected=$5 + local requested_version=$2 + local snap_present=$3 + local dpkg_present=$4 + local rpm_present=$5 + local expected=$6 local actual actual="$( + export OPENSHELL_VERSION="$requested_version" has_cmd() { case "$1" in snap) [ "$snap_present" = "1" ] ;; @@ -125,9 +127,16 @@ assert_linux_package_method() { fi } -assert_linux_package_method "snap takes precedence over deb and rpm" 1 1 1 snap -assert_linux_package_method "deb is selected without snap" 0 1 1 deb -assert_linux_package_method "rpm is selected without snap or deb" 0 0 1 rpm +assert_linux_package_method "snap takes precedence over deb and rpm" "" 1 1 1 snap +assert_linux_package_method "dev uses snap" dev 1 1 1 snap +assert_linux_package_method "pre uses deb despite snap" pre 1 1 1 deb +assert_linux_package_method "numbered prerelease uses deb despite snap" v0.1.0-pre.3 1 1 1 deb +assert_linux_package_method "pre uses rpm despite snap" pre 1 0 1 rpm +assert_linux_package_method "pinned stable uses deb despite snap" v1.2.3 1 1 1 deb +assert_linux_package_method "pinned stable uses rpm despite snap" v1.2.3 1 0 1 rpm +assert_linux_package_method "deb is selected without snap" "" 0 1 1 deb +assert_linux_package_method "dev uses deb without snap" dev 0 1 1 deb +assert_linux_package_method "rpm is selected without snap or deb" "" 0 0 1 rpm if ! ( find_existing_native_openshell_bin() { return 1; } @@ -223,22 +232,15 @@ if [ -s "$err" ]; then exit 1 fi -for requested_version in pre v1.2.3; do - if ! OPENSHELL_VERSION="$requested_version" openshell_snap_channel >"$out" 2>"$err"; then - echo "FAIL: '${requested_version}' must select the latest/stable Snap channel" >&2 - cat "$err" >&2 || true - exit 1 - fi - if [ "$(cat "$out")" != "latest/stable" ]; then - echo "FAIL: '${requested_version}' must select the latest/stable Snap channel" >&2 - exit 1 - fi - if ! grep -Fq "OPENSHELL_VERSION=${requested_version} is ignored for Snap installs" "$err"; then - echo "FAIL: '${requested_version}' must warn that the requested version is ignored" >&2 - cat "$err" >&2 || true - exit 1 - fi -done +if (OPENSHELL_VERSION=v1.2.3 openshell_snap_channel) >"$out" 2>"$err"; then + echo "FAIL: pinned release must not select a Snap channel" >&2 + exit 1 +fi +if ! grep -Fq "Snap installs do not support OPENSHELL_VERSION=v1.2.3" "$err"; then + echo "FAIL: pinned release Snap rejection was not explained" >&2 + cat "$err" >&2 + exit 1 +fi rm -f "$out" "$err" assert_snap_install_flow() { @@ -299,7 +301,7 @@ wait:gateway-status" assert_snap_install_flow \ "existing OpenShell snap is refreshed" \ - 1 1 pre \ + 1 1 "" \ "wait:docker root:snap refresh openshell --channel=latest/stable ensure:gateway-config @@ -502,19 +504,21 @@ gh() { ;; api:*) case "$*" in - *"?name="*) printf '123456\n' ;; - *"actions/workflows/release-tag.yml/runs?status=success"*) - printf '%s\n' 100 101 + *"git/matching-refs/tags/v"*) + printf '%s\n' v2.0.0-pre.1 v1.0.0-pre.2 v0.1.0-pre.9 + ;; + *"?name=openshell-v2.0.0-pre.1-linux-amd64-deb"*) + [ "${MOCK_NO_PRERELEASE:-0}" = "1" ] || printf '999\n' + ;; + *"?name=openshell-v1.0.0-pre.2-linux-amd64-deb"*) + [ "${MOCK_NO_PRERELEASE:-0}" = "1" ] || printf '101\n' ;; + *"?name=openshell-v0.1.0-pre.9-linux-amd64-deb"*) + [ "${MOCK_NO_PRERELEASE:-0}" = "1" ] || printf '123456\n' + ;; + *"actions/runs/999"*) printf 'false\n' ;; + *"actions/runs/101"*) printf 'true\n' ;; *) - if [ "${MOCK_NO_PRERELEASE:-0}" != "1" ]; then - printf '%b\n' \ - '100\topenshell-v0.1.0-pre.9-linux-amd64-deb' \ - '101\topenshell-v1.0.0-pre.2-linux-amd64-deb' \ - '101\topenshell-v1.0.0-pre.1-macos-arm64' \ - '101\topenshell-v0.2.0-pre.10-linux-aarch64-rpm' \ - '999\topenshell-v2.0.0-pre.1-linux-amd64-deb' - fi ;; esac ;; @@ -539,13 +543,29 @@ if [ "$resolved_prerelease" != "v1.0.0-pre.2" ]; then echo "FAIL: pre alias resolved to ${resolved_prerelease}, expected v1.0.0-pre.2" >&2 exit 1 fi -if ! grep -Fq 'actions/workflows/release-tag.yml/runs?status=success' "$mock_gh_log"; then - echo "FAIL: pre alias did not query successful Release Tag workflow runs" >&2 +if ! grep -Fq 'git/matching-refs/tags/v' "$mock_gh_log"; then + echo "FAIL: pre alias did not query prerelease tags" >&2 + cat "$mock_gh_log" >&2 + exit 1 +fi +if ! grep -Fq 'actions/artifacts?name=openshell-v1.0.0-pre.2-linux-amd64-deb' "$mock_gh_log"; then + echo "FAIL: pre alias did not query the platform artifact by name" >&2 + cat "$mock_gh_log" >&2 + exit 1 +fi +if ! grep -Fq 'actions/runs/999' "$mock_gh_log" || + ! grep -Fq 'actions/runs/101' "$mock_gh_log"; then + echo "FAIL: pre alias did not skip an unsuccessful run" >&2 + cat "$mock_gh_log" >&2 + exit 1 +fi +if ! grep -Fq '.conclusion == "success"' "$mock_gh_log"; then + echo "FAIL: pre alias did not require a successful workflow run" >&2 cat "$mock_gh_log" >&2 exit 1 fi -if ! grep -Fq 'select(.status == "completed" and .conclusion == "success")' "$mock_gh_log"; then - echo "FAIL: pre alias did not require completed successful workflow runs" >&2 +if grep -Fq -- '--paginate' "$mock_gh_log"; then + echo "FAIL: pre alias must not scan every workflow run or artifact" >&2 cat "$mock_gh_log" >&2 exit 1 fi From 48725c5fcc35bff778dae3fe6991a6ddacfa3c95 Mon Sep 17 00:00:00 2001 From: Jim Meyer Date: Fri, 25 Sep 2026 01:26:59 +0000 Subject: [PATCH 6/7] ci(release): move CodeQL, Trivy, and Zizmor to advisory (#3693) * ci(release): Move CodeQL, Trivy, and Zizmor to advisory * docs(ci): describe advisory static findings for tagged releases Signed-off-by: Jim Meyer --------- Signed-off-by: Jim Meyer --- .github/workflows/release-tag.yml | 2 +- CI.md | 11 +++++++---- architecture/build.md | 6 ++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 98fd1394e1..c569a2a615 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -142,7 +142,7 @@ jobs: with: candidate_ref: ${{ inputs.tag || github.ref_name }} fail-on-codex-findings: true - fail-on-static-findings: true + fail-on-static-findings: false images: | ghcr.io/nvidia/openshell/gateway:${{ needs.compute-versions.outputs.source_sha }} ghcr.io/nvidia/openshell/sandbox:${{ needs.compute-versions.outputs.source_sha }} diff --git a/CI.md b/CI.md index 72f63340ff..31d0b097b3 100644 --- a/CI.md +++ b/CI.md @@ -103,10 +103,13 @@ Actions or call it from another workflow. All applicable children analyze the candidate snapshot. Cargo Deny uses its existing NVIDIA self-hosted runner and CI container. -Tagged releases treat CodeQL, Trivy, Zizmor, Cargo Deny, and Codex Security -findings as failures of the currently implemented qualification profile. A -profile failure does not prevent a pre-release candidate's complete artifact -set from being published, but it does prevent stable publication. +Tagged releases treat Cargo Deny and Codex Security findings as failures of the +currently implemented qualification profile. A profile failure does not prevent +a pre-release candidate's complete artifact set from being published, but it +does prevent stable publication. CodeQL, Trivy, and Zizmor findings are +temporarily informational for tagged releases: the existing findings were +reviewed and accepted for v0.1.0 and will be addressed in 0.1.x releases. +Scanner failures still fail qualification. ```shell gh workflow run security-scan.yml --ref main \ diff --git a/architecture/build.md b/architecture/build.md index 6b5077d966..d1faa3ea53 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -536,8 +536,10 @@ RFC's complete qualification coverage. The tagged release workflow calls the aggregate Security Scan after publishing the candidate's commit-addressed gateway, sandbox, and supervisor images. CodeQL, Trivy, Cargo Deny, and Actionlint/Zizmor run for every release tag; Codex Security -also runs for pre-release tags. High or Critical findings and scanner failures -fail qualification. +also runs for pre-release tags. Scanner failures, Cargo Deny advisories, and +High or Critical Codex Security findings fail qualification. CodeQL, Trivy, and +Zizmor findings are temporarily informational while the findings accepted for +v0.1.0 are addressed in 0.1.x releases. The `Release Qualification` job aggregates security, conformance, feature, Docker E2E, and VM E2E results. The currently implemented profile gates stable From 1374672967c002400f43c0e9c891f7429ccfd236 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 24 Sep 2026 18:55:45 -0700 Subject: [PATCH 7/7] fix(helm): restore Kubernetes e2e chart rendering (#3692) Signed-off-by: Drew Newberry --- deploy/helm/openshell/ci/values-skaffold.yaml | 3 --- tasks/helm.toml | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 8def2796a4..4c017f669e 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -17,6 +17,3 @@ sandbox: supervisor: image: pullPolicy: if_not_present - # The local k3s cluster created by `mise run helm:k3s:create` enables its - # built-in NetworkPolicy controller for sandbox namespaces. - sandboxRuntime: diff --git a/tasks/helm.toml b/tasks/helm.toml index 43f01bf3f1..e2a4e001ce 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -40,11 +40,13 @@ run = """ echo "--- helm lint: defaults ---" echo "values files: deploy/helm/openshell/values.yaml" helm lint deploy/helm/openshell --set agentSandbox.preflight.enabled=false + helm template openshell deploy/helm/openshell --set agentSandbox.preflight.enabled=false >/dev/null for f in deploy/helm/openshell/ci/values-*.yaml; do variant=$(basename "$f" .yaml | sed 's/values-//') echo "--- helm lint: $variant ---" echo "values files: deploy/helm/openshell/values.yaml, $f" helm lint deploy/helm/openshell -f "$f" --set agentSandbox.preflight.enabled=false + helm template openshell deploy/helm/openshell -f "$f" --set agentSandbox.preflight.enabled=false >/dev/null done echo "--- helm lint: workspace defaults ---" helm lint deploy/helm/openshell-workspace