diff --git a/README.md b/README.md index d441d5bcf9..6049b609a3 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,16 @@ [![PyPI](https://img.shields.io/badge/PyPI-openshell-orange?logo=pypi)](https://pypi.org/project/openshell/) [![Security Policy](https://img.shields.io/badge/Security-Report%20a%20Vulnerability-red)](SECURITY.md) [![Documentation](https://img.shields.io/badge/docs-latest-brightgreen)](https://docs.nvidia.com/openshell/latest/index.html) -[![Project Status](https://img.shields.io/badge/status-alpha-orange)](https://docs.nvidia.com/openshell/latest/about/release-notes.html) +[![Project Status](https://img.shields.io/badge/status-alpha-orange)](https://github.com/NVIDIA/OpenShell/releases) > [!IMPORTANT] -> **OpenShell 0.1.0 is coming soon.** [Track progress in the 0.1.0 milestone](https://github.com/NVIDIA/OpenShell/milestone/10), [read the prerelease documentation](https://docs.nvidia.com/openshell/dev/index.html), or [install a prerelease](#prerelease-and-development-builds). +> **New in OpenShell 0.1.0:** a stable release cadence, an improved security model, an expanded extension surface, and new APIs. [Read the 0.1.0 upgrade guide](https://docs.nvidia.com/openshell/latest/upgrade/0-1-0). OpenShell is the safe, private runtime for autonomous AI agents. It provides sandboxed execution environments that protect your data, credentials, and infrastructure — governed by declarative YAML policies that prevent unauthorized file access, data exfiltration, and uncontrolled network activity. OpenShell is built agent-first. It ships public agent skills for using and operating OpenShell, plus separate repository-aware workflows for contributors and maintainers. -## Quickstart +## Install OpenShell ### Prerequisites @@ -190,20 +190,13 @@ Docker-backed GPU sandboxes auto-select CDI when available and otherwise fall ba **Requirements:** NVIDIA drivers and the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) must be installed on the host. The sandbox image itself must include the appropriate GPU drivers and libraries for your workload — the default Ubuntu image does not. See the [BYOC example](https://github.com/NVIDIA/OpenShell/tree/main/examples/bring-your-own-container) for building a custom sandbox image with GPU support. -## Supported Agents +## Running Agents OpenShell can run Linux agents packaged in OCI images. The default Ubuntu workload does not bundle agent CLIs. Build or select an image containing your -agent, then authorize its binary paths, service endpoints, and credentials. - -| Agent | Integration | -| ----- | ----------- | -| [Claude Code](https://docs.anthropic.com/en/docs/claude-code) | Package Claude Code in a workload image and attach a `claude-code` provider or another endpoint-bearing model profile. | -| [OpenCode](https://opencode.ai/) | Package OpenCode in a workload image and attach its model provider and policy. | -| [Codex](https://developers.openai.com/codex) | Package Codex in a workload image and attach an OpenAI provider and policy. | -| [GitHub Copilot CLI](https://docs.github.com/en/copilot/github-copilot-in-the-cli) | Package the CLI in a workload image and attach GitHub credentials and policy. | -| [OpenClaw](https://openclaw.ai/) | Use the [NemoClaw](https://github.com/NVIDIA/NemoClaw) blueprint. | -| [Hermes Agent](https://github.com/NousResearch/hermes-agent) | Use the [NemoClaw](https://github.com/NVIDIA/NemoClaw) blueprint. | +agent, then authorize its binary paths, service endpoints, and credentials. See +[Run Your First Agent](https://docs.nvidia.com/openshell/latest/about/run-an-agent) +for the image, provider, and policy workflow. ## Key Commands @@ -284,7 +277,7 @@ Agent implementation is human-directed: a user may request a phase directly, or ## Learn More - [Full Documentation](https://docs.nvidia.com/openshell/latest/index.html) — overview, architecture, tutorials, and reference -- [Quickstart](https://docs.nvidia.com/openshell/latest/get-started/quickstart) — detailed install and first sandbox walkthrough +- [Run Your First Agent](https://docs.nvidia.com/openshell/latest/about/run-an-agent) — prepare an image, attach providers, and launch an agent - [GitHub Sandbox Tutorial](https://docs.nvidia.com/openshell/latest/get-started/tutorials/github-sandbox) — end-to-end scoped GitHub repo access - [Architecture](https://github.com/NVIDIA/OpenShell/tree/main/architecture) — detailed architecture docs and design decisions - [Roadmap](https://github.com/orgs/NVIDIA/projects/233) — planned work and project priorities diff --git a/TESTING.md b/TESTING.md index 42495fc23a..9088be5261 100644 --- a/TESTING.md +++ b/TESTING.md @@ -359,9 +359,40 @@ TAG=0.0.115 skopeo inspect "docker://ghcr.io/nvidia/openshell/gateway:${TAG}" ``` -`IMAGE_TAG` sets only the gateway/supervisor image; the CLI under test is always -built from your branch. To validate against images from your exact commit -instead, build and push them and point `OPENSHELL_REGISTRY`/`IMAGE_TAG` at them. +`IMAGE_TAG` sets the default tag for the gateway/supervisor image pair; the CLI +under test is always built from your branch. To validate against images from +your exact commit instead, build and push them and point +`OPENSHELL_REGISTRY`/`IMAGE_TAG` at them. + +Test wrappers accept independent image overrides: + +```shell +GATEWAY_IMAGE=registry.example.com/custom/gateway:test \ +SUPERVISOR_IMAGE=registry.example.com/custom/supervisor:test \ +SANDBOX_IMAGE=registry.example.com/custom/sandbox:test \ +mise run e2e:kubernetes +``` + +`GATEWAY_IMAGE` applies to the Kubernetes gateway container. `SUPERVISOR_IMAGE` +applies to the trusted supervisor image selected by the Kubernetes, Docker, and +Podman wrappers. `SANDBOX_IMAGE` applies to the trusted workload-side runtime +image that stages the `openshell-sandbox` binary. A repository-only value +inherits `IMAGE_TAG`; a value with an explicit tag or `@sha256:` digest is used +as-is. When these variables are unset, the existing `OPENSHELL_REGISTRY` plus +`IMAGE_TAG` behavior is retained. +The Docker and Podman wrappers continue to give +`OPENSHELL_DOCKER_SUPERVISOR_IMAGE` and `OPENSHELL_SUPERVISOR_IMAGE` precedence +over `SUPERVISOR_IMAGE`. + +Digest-pinned Kubernetes overrides require disabling local image builds, because +Docker cannot tag a locally built image with a digest reference: + +```shell +OPENSHELL_E2E_KUBE_BUILD_IMAGES=0 \ +GATEWAY_IMAGE=registry.example.com/custom/gateway@sha256: \ +SUPERVISOR_IMAGE=registry.example.com/custom/supervisor@sha256: \ +mise run e2e:kubernetes +``` Available task variants: @@ -386,6 +417,9 @@ Kubernetes e2e environment variables: | `OPENSHELL_E2E_KUBERNETES_FEATURES` | Cargo feature flags (default: `e2e,e2e-host-gateway,e2e-kubernetes`) | | `IMAGE_TAG` | Gateway/supervisor image tag (default: `latest` for existing clusters) | | `OPENSHELL_REGISTRY` | Image registry prefix (default: `ghcr.io/nvidia/openshell`) | +| `GATEWAY_IMAGE` | Kubernetes gateway image repository or complete tagged/digest-pinned image reference; digests require `OPENSHELL_E2E_KUBE_BUILD_IMAGES=0` | +| `SUPERVISOR_IMAGE` | Gateway/supervisor image repository or complete tagged/digest-pinned image reference; Kubernetes digests require `OPENSHELL_E2E_KUBE_BUILD_IMAGES=0` | +| `SANDBOX_IMAGE` | Trusted sandbox runtime image repository or complete tagged/digest-pinned image reference | Run a single test directly with cargo: diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 53d24aa0cc..a341ef2ba2 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -408,6 +408,30 @@ and explicit `/sandbox` values select `/sandbox`; other paths must already exist without symlink or reserved-mount collisions and must be usable by the resolved identity. Kubernetes and VM use `/sandbox`. +### Executable Identity Binding + +Every mediated network open carries a connection-bound `BinaryIdentity` from +the isolation backend. The identity contains the socket-owning executable and +each executable ancestor, nearest first, as an absolute workload path plus a +SHA-256 digest. The backend resolves these values for the accepted connection +and hashes already-open live executable objects rather than reopening their +paths. Command-line paths remain diagnostic context and cannot authorize a +request. + +Before policy evaluation, the supervisor validates and pins the complete leaf +and ancestor chain in one runtime-scoped trust-on-first-use cache. It rejects +missing digests, invalid paths, conflicting evidence within a chain, or a +digest that differs from an existing path pin. Validation and insertion are +atomic, so a rejected chain cannot leave partial pins. + +The cache is shared across authorization paths for the lifetime of the +supervisor network runtime. Policy reloads replace policy state without +clearing executable pins; restarting the runtime creates a new cache. OPA +receives executable and ancestor paths plus endpoint policy context. Digests +remain supervisor-side integrity evidence and are not policy inputs. Any +unavailable, incomplete, or conflicting executable evidence fails closed +before OPA can authorize the connection. + The Kubernetes driver creates the namespace-wide empty-egress workload fence before a suspended Sandbox CR, then provisions split immutable bootstrap Secrets, the private runtime Service, and a gated supervisor Pod. A @@ -465,11 +489,12 @@ management. RBAC uses a namespace-scoped Role. **Managed** auto-creates a K8s namespace per workspace on first sandbox create. Each new namespace receives a ServiceAccount and the configured gateway-only -SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the -driver's source namespace on every sandbox create so registry credential -rotations propagate. Their names are operator-selected gateway configuration, -not caller attachments. Copies carry gateway and workspace ownership labels; an -unrelated existing target is never adopted. The namespace also copies +SSH ingress NetworkPolicy. Each sandbox runtime generation gets immutable copies +of the configured image-pull Secrets, read from the driver's source namespace and +named after the generation, so a sandbox picks up rotated registry credentials +on its next start. Their sources are operator-selected gateway configuration, +not caller attachments. An existing Secret with a generation name fails the +create and is never adopted. The namespace also copies OpenShift SCC UID-range and supplemental-group annotations from the gateway namespace when present. The driver deletes the namespace during workspace deletion. The workspace remains durably `Terminating` until the Kubernetes API @@ -481,14 +506,15 @@ DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-charac limit. RBAC promotes sandbox CRD permissions to a ClusterRole and adds namespace `create`/`delete` and ServiceAccount `create`/`get` permissions. -Secret copies use server-side apply. Kubernetes authorizes an apply to an -existing Secret as `patch`, but also requires `create` authorization when the -target does not exist. RBAC cannot constrain `create` by `resourceNames`, so -managed mode grants cluster-wide Secret `create`, `list`, and `delete` for -generation-scoped bootstrap Secret creation and rollback recovery while keeping -source reads and subsequent patches restricted to the explicitly configured TLS -and image-pull Secret names. Recovery lists bootstrap Secrets by sandbox and -component labels, then deletes stale generations with UID preconditions. The +Outside shared mode, the gateway client TLS material is staged into each +generation's supervisor bootstrap Secret rather than mounted from a Secret in +the workspace namespace. Every Secret the driver writes into a workspace +namespace is therefore generation-scoped, immutable, and created with `create` +only. RBAC cannot constrain `create` by `resourceNames`, so managed mode grants +cluster-wide Secret `create` and `delete`; source reads use a Role in the +driver's source namespace. Recovery deletes the Secrets of the recorded and +target runtime generations by exact name; Pod owner references let garbage +collection remove any other generation. The driver exercises these broad permissions only in gateway-owned managed namespaces. This depends on the managed-mode ownership invariant described below; the gateway ServiceAccount must not be shared with unrelated workloads. @@ -496,6 +522,9 @@ the gateway ServiceAccount must not be shared with unrelated workloads. Operator mode does not create NetworkPolicies or copy image-pull Secrets. Platform teams must apply the gateway ingress boundary and provision configured image-pull Secrets in every operator-managed namespace. +The gateway ClusterRole grants no Secret permissions in operator mode. The +`openshell-workspace` chart Role installed in each operator-managed namespace +grants bootstrap Secret `create` and `delete`. **Operator** uses pre-provisioned namespaces discovered through two optional sources: a K8s label selector (`operator_namespace_label`) and a drop-in @@ -555,9 +584,10 @@ isolation boundary. ### Credential Driver Integration The Kubernetes Secrets credential driver (`openshell-driver-kubernetes-secrets`) -stores secrets in workspace-specific namespaces when `workspace_mode` is managed -or operator. In shared mode, all secrets render into the single configured -namespace. +stores every provider credential in its single configured namespace, in every +workspace mode, and rejects handles that reference another namespace. The +gateway reaches those Secrets through a namespaced Role; the gateway +ClusterRole grants no credential Secret permissions. When runtime infrastructure changes, validate the relevant sandbox e2e path and update the matching driver README if a maintainer-facing constraint changes. diff --git a/architecture/gateway.md b/architecture/gateway.md index 3de3154033..689bd4b493 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -342,6 +342,8 @@ driver/runtime identity recorded at provisioning before returning the current generation-bound session JWT. Session authentication checks the durable runtime generation and token lineage for every sandbox RPC, so a replaced runtime and legacy unbound tokens cannot retain provider or control-plane access. The +gateway admits only explicitly typed, generation-bound session JWTs for sandbox +RPCs. It does not accept the pre-session untyped JWT format. The Kubernetes driver uses its own named configuration to run TokenReview and verify the live pod and controlling Sandbox CR. Its runtime identity binds the namespace, immutable Sandbox CR UID, and supervisor Pod UID. Restart preserves @@ -356,11 +358,8 @@ can recover that same successor for 30 seconds when the request matches, but it cannot authorize ordinary RPCs or choose another successor. Advancing the successor removes that retry path across every gateway replica. Short `gateway_jwt.ttl_secs` lifetimes still bound the exposure of a current bearer -that has not yet been refreshed. -Omitting `gateway_jwt.ttl_secs` selects non-expiring tokens for local -single-player Docker, Podman, and VM gateways; those tokens carry `exp = 0`. -Kubernetes and other shared deployments should set a positive TTL. Explicit -zero is rejected. +that has not yet been refreshed. Omitting `gateway_jwt.ttl_secs` uses a +900-second lifetime. Explicit zero is rejected. Gateway JWT signing-key rotation is currently an offline operator action. The runtime loads one active signing key and one matching public verification key diff --git a/architecture/sandbox-limits.md b/architecture/sandbox-limits.md index b8d03db39d..049aa5d99d 100644 --- a/architecture/sandbox-limits.md +++ b/architecture/sandbox-limits.md @@ -126,6 +126,7 @@ can contain a valid sequence gap. | Path | Current bound | Terminal behavior | |---|---:|---| +| Executable identity pins | 4,096 unique paths per supervisor lifetime | Reject an entire identity chain before insertion when its new paths would exceed the bound. Existing pins remain usable and are never evicted. Staged TCP reports resource exhaustion. | | Corporate proxy CONNECT response headers | 8 KiB | Fail the tunnel. | | Corporate proxy CONNECT handshake | 30 s total | Fail the tunnel; validated-address attempts share the aggregate budget. | | Token-grant HTTP request | 30 s request and connect | Fail credential resolution. | diff --git a/architecture/sandbox.md b/architecture/sandbox.md index f68c1b67a7..c2de9a1d6d 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -27,6 +27,12 @@ TCP Service, or VM vsock channel. Independent bidirectional `Exchange` RPCs carry lifecycle, exec, TCP, and forwarding traffic, while one persistent bidirectional `Mediate` RPC carries multiplexed DNS traffic. General application UDP is unsupported; UDP DNS remains mediated by the supervisor. +Both ends size HTTP/2 flow control so the connection window exceeds the +per-stream window times the concurrent-stream limit plus a reserve. Relays whose +workload stops reading therefore stall only their own streams and cannot starve +DNS, exec, or control traffic of connection-level credit. The supervisor caps +concurrent TCP relay and pending-accept streams below the stream limit, so new +workload connections queue before control exchanges lose stream slots. The sandbox probes HTTP/2 connection liveness every five seconds and closes connections that miss a ten-second acknowledgement deadline. Closing a connection freezes the owned workload process tree and cancels its stream @@ -39,6 +45,12 @@ credentials to claim the existing runtime generation. Confirmation resumes the workload; expiration terminates it. A credential replacement does not displace the active connection until the new connection is confirmed. Idle healthy connections remain usable. +A failed stream alone does not trigger reconnection. The supervisor first +reconfirms on the current connection and keeps it if the boundary answers. +Otherwise it closes that transport before replaying attach, and retries while +the sandbox still reports the equal-epoch connection as active, until the +boundary observes the disconnect. The sandbox reports attach and confirm +rejections as typed errors rather than closing the stream. TCP mediation accepts use the same authenticated transport recovery as process waits. A healthy idle accept has no timeout. An interrupted pending open fails closed, while a replacement accept waits for new workload traffic; decisions and established byte streams are not replayed. Boundary rejections and failed recovery remain terminal to the proxy. A renewed Sandbox Protocol bearer is authenticated even when its credential epoch is unchanged. The supervisor confirms that bearer on the active physical connection and records its fingerprint only after confirmation succeeds, preserving pending streams and the mediation session. Changing the credential epoch still requires an authenticated replacement connection. diff --git a/crates/openshell-binary-identity/README.md b/crates/openshell-binary-identity/README.md index 9f11201b7e..472ddcd41a 100644 --- a/crates/openshell-binary-identity/README.md +++ b/crates/openshell-binary-identity/README.md @@ -6,12 +6,15 @@ Docker obtains an authoritative thread ID from seccomp notification, while the co-located Linux path maps an accepted socket to its owning processes. Given an authoritative Linux PID and an optional trusted process-tree root, the -crate reads the executable path from procfs, hashes the live `/proc//exe` -object, and collects bounded executable ancestry and diagnostic command-line -paths. Resolution failures are returned as `ResolveError` so the caller can -deny the associated connection. +crate opens the leaf and bounded ancestor `/proc//exe` objects, hashes +those already-open live objects, and returns path-and-digest evidence for the +whole executable chain. It also collects diagnostic command-line paths, which +never authorize network access. Resolution fails without returning a partial +identity if any executable cannot be opened, hashed, or validated after +hashing. The caller receives `ResolveError` and denies the associated +connection. The crate does not intercept connections, authenticate remote observers, or evaluate policy. The isolation backend remains responsible for binding the resolved identity to the active boundary and exact accepted connection before -constructing `MediatedConnection`. +constructing `PendingTcpOpen`. diff --git a/crates/openshell-binary-identity/src/lib.rs b/crates/openshell-binary-identity/src/lib.rs index eb06934eaf..ed4dde362a 100644 --- a/crates/openshell-binary-identity/src/lib.rs +++ b/crates/openshell-binary-identity/src/lib.rs @@ -9,10 +9,10 @@ //! collects its process ancestry. Backends bind the returned identity to the //! intercepted connection before constructing a `MediatedConnection`. -#[cfg(target_os = "linux")] -use openshell_isolation_interface::contract::Sha256Digest; use openshell_isolation_interface::contract::{BinaryIdentity, ResolveError}; #[cfg(target_os = "linux")] +use openshell_isolation_interface::contract::{ExecutableIdentity, Sha256Digest}; +#[cfg(target_os = "linux")] use std::collections::HashMap; #[cfg(target_os = "linux")] use std::sync::{Arc, Mutex}; @@ -99,24 +99,19 @@ fn resolve_linux_process( cache: &Mutex>, ) -> Result { let (snapshot, mut executable) = open_process_snapshot(pid)?; - let binary_path = snapshot.binary_path.clone(); - let executable_key = snapshot.executable_cache_key(); - let cached_digest = cached_executable_digest(cache, executable_key); - let binary_digest = cached_digest.map_or_else(|| hash_executable(pid, &mut executable), Ok)?; - let ancestor_processes = collect_ancestor_processes(&snapshot, ancestor_root); - let ancestors = ancestor_processes + let mut ancestor_processes = collect_ancestor_processes(&snapshot, ancestor_root)?; + + let mut excluded_paths = ancestor_processes .iter() - .map(|snapshot| snapshot.binary_path.clone()) + .map(|(snapshot, _)| snapshot.binary_path.clone()) .collect::>(); - - let mut excluded_paths = ancestors.clone(); - excluded_paths.push(binary_path.clone()); + excluded_paths.push(snapshot.binary_path.clone()); let cmdline_paths = cmdline_absolute_paths(&snapshot.cmdline) .into_iter() .chain( ancestor_processes .iter() - .flat_map(|snapshot| cmdline_absolute_paths(&snapshot.cmdline)), + .flat_map(|(snapshot, _)| cmdline_absolute_paths(&snapshot.cmdline)), ) .filter(|path| !excluded_paths.contains(path)) .fold(Vec::new(), |mut paths, path| { @@ -126,22 +121,55 @@ fn resolve_linux_process( paths }); + let (executable_identity, pending_cache_entry) = + resolve_open_executable(&snapshot, &mut executable, cache)?; + let mut pending_cache_entries = pending_cache_entry.into_iter().collect::>(); + let mut ancestors = Vec::with_capacity(ancestor_processes.len()); + for (ancestor, executable) in &mut ancestor_processes { + let (identity, pending_cache_entry) = resolve_open_executable(ancestor, executable, cache)?; + ancestors.push(identity); + pending_cache_entries.extend(pending_cache_entry); + } + validate_process_snapshot(pid, &snapshot)?; - for ancestor in &ancestor_processes { + for (ancestor, _) in &ancestor_processes { validate_process_snapshot(ancestor.pid, ancestor)?; } - if cached_digest.is_none() { - cache_executable_digest(cache, executable_key, binary_digest); + for (key, digest) in pending_cache_entries { + cache_executable_digest(cache, key, digest); } Ok(BinaryIdentity { - binary_path, - binary_digest: Some(binary_digest), + executable: executable_identity, ancestors, cmdline_paths, }) } +#[cfg(target_os = "linux")] +fn resolve_open_executable( + snapshot: &ProcessSnapshot, + executable: &mut std::fs::File, + cache: &Mutex>, +) -> Result< + ( + ExecutableIdentity, + Option<(ExecutableCacheKey, Sha256Digest)>, + ), + ResolveError, +> { + let key = snapshot.executable_cache_key(); + let cached_digest = cached_executable_digest(cache, key); + let digest = cached_digest.map_or_else(|| hash_executable(snapshot.pid, executable), Ok)?; + Ok(( + ExecutableIdentity { + path: snapshot.binary_path.clone(), + digest: Some(digest), + }, + cached_digest.is_none().then_some((key, digest)), + )) +} + #[cfg(target_os = "linux")] #[derive(Debug, PartialEq, Eq)] struct ProcessSnapshot { @@ -351,21 +379,17 @@ fn hash_executable(pid: u32, executable: &mut std::fs::File) -> Result, -) -> Vec { +) -> Result, ResolveError> { const MAX_DEPTH: usize = 64; if ancestor_root == Some(process.pid) { - return Vec::new(); + return Ok(Vec::new()); } - let mut ancestors = Vec::new(); + let mut ancestors: Vec<(ProcessSnapshot, std::fs::File)> = Vec::new(); let mut parent = process.parent_pid; for _ in 0..MAX_DEPTH { - if parent == 0 - || ancestors - .iter() - .any(|current: &ProcessSnapshot| current.pid == parent) - { + if parent == 0 || ancestors.iter().any(|(current, _)| current.pid == parent) { break; } @@ -375,17 +399,15 @@ fn collect_ancestor_processes( break; } - let Ok((snapshot, _executable)) = open_process_snapshot(parent) else { - break; - }; + let (snapshot, executable) = open_process_snapshot(parent)?; let next_parent = snapshot.parent_pid; - ancestors.push(snapshot); + ancestors.push((snapshot, executable)); if ancestor_root == Some(parent) || parent == 1 { break; } parent = next_parent; } - ancestors + Ok(ancestors) } #[cfg(target_os = "linux")] @@ -440,6 +462,15 @@ fn cmdline_absolute_paths(cmdline: &[u8]) -> Vec { mod tests { use super::*; + struct ChildGuard(std::process::Child); + + impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + #[cfg(target_os = "linux")] #[test] fn resolver_cache_is_owned_and_only_explicit_clones_share_it() { @@ -459,8 +490,34 @@ mod tests { .resolve(std::process::id()) .expect("resolve current process"); - assert!(identity.binary_path.is_absolute()); - assert!(identity.binary_digest.is_some()); + assert!(identity.executable.path.is_absolute()); + assert!(identity.executable.digest.is_some()); + } + + #[test] + #[ignore = "subprocess fixture for executable identity tests"] + fn identity_helper_process() { + std::thread::sleep(std::time::Duration::from_secs(30)); + } + + #[test] + fn resolves_child_and_hashes_ancestor_chain() { + let parent_pid = std::process::id(); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--ignored", "--exact", "tests::identity_helper_process"]) + .spawn() + .expect("spawn child"); + let child = ChildGuard(child); + + let identity = ProcfsIdentityResolver::for_process_tree(parent_pid) + .resolve(child.0.id()) + .expect("resolve child identity"); + + assert!(identity.executable.path.is_absolute()); + assert!(identity.executable.digest.is_some()); + let parent = identity.ancestors.last().expect("process-tree root"); + assert_eq!(parent.path, std::env::current_exe().unwrap()); + assert!(parent.digest.is_some()); } #[test] diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 3d8b6eda8b..1c844864a4 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -800,16 +800,18 @@ pub struct GatewayJwtConfig { /// `openshell`. #[serde(default = "default_gateway_id")] pub gateway_id: String, - /// Token lifetime in seconds. Omit the field for a non-expiring token. + /// Token lifetime in seconds. Defaults to 15 minutes when omitted. /// Explicit zero is invalid. #[serde(default, skip_serializing_if = "Option::is_none")] pub ttl_secs: Option, } impl GatewayJwtConfig { - /// Effective token lifetime. `None` represents a non-expiring token. - pub fn sandbox_token_ttl(&self) -> Option { - self.ttl_secs.map(|ttl| Duration::from_secs(ttl.get())) + /// Effective token lifetime. + pub fn token_ttl(&self) -> Duration { + self.ttl_secs.map_or(Duration::from_mins(15), |ttl| { + Duration::from_secs(ttl.get()) + }) } } @@ -1213,7 +1215,7 @@ mod tests { } #[test] - fn gateway_jwt_ttl_defaults_to_non_expiring() { + fn gateway_jwt_ttl_defaults_to_fifteen_minutes() { let cfg: GatewayJwtConfig = serde_json::from_value(serde_json::json!({ "signing_key_path": "/tmp/signing.pem", "public_key_path": "/tmp/public.pem", @@ -1222,7 +1224,7 @@ mod tests { .expect("gateway JWT config should deserialize with default ttl"); assert_eq!(cfg.ttl_secs, None); - assert_eq!(cfg.sandbox_token_ttl(), None); + assert_eq!(cfg.token_ttl(), Duration::from_mins(15)); let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); assert!(serialized.get("ttl_secs").is_none()); @@ -1238,7 +1240,7 @@ mod tests { })) .expect("gateway JWT config should deserialize with positive ttl"); - assert_eq!(cfg.sandbox_token_ttl(), Some(Duration::from_hours(1))); + assert_eq!(cfg.token_ttl(), Duration::from_hours(1)); let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); assert_eq!(serialized["ttl_secs"], 3600); } diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 5304286fd2..cbd6de5670 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -648,8 +648,7 @@ async fn refresh_extension_credentials_with_client( /// Compute the next refresh delay: 80 % of the time remaining until the /// current token's `exp`, plus up to 10 % jitter, with a small lower bound /// for already-expired tokens and capped at 12 h. If the token can't be parsed -/// (legacy/non-JWT bearer) or carries the `exp = 0` non-expiring sentinel, -/// default to 6 h. +/// (for example, an opaque bootstrap bearer), default to 6 h. fn compute_refresh_delay(slot: &TokenSlot) -> Duration { let token = slot .read() @@ -663,17 +662,14 @@ fn compute_refresh_delay(slot: &TokenSlot) -> Duration { .map_or(0, |d| d.as_millis()), ) .unwrap_or(i64::MAX); - let mut delay_ms = match parse_jwt_exp_ms(bearer) { - Some(0) | None => 21_600_000, - Some(exp) => { - let remaining_ms = exp - now_ms; - if remaining_ms <= 0 { - 1_000 - } else { - (remaining_ms * 8 / 10).clamp(1_000, 43_200_000) - } + let mut delay_ms = parse_jwt_exp_ms(bearer).map_or(21_600_000, |exp| { + let remaining_ms = exp - now_ms; + if remaining_ms <= 0 { + 1_000 + } else { + (remaining_ms * 8 / 10).clamp(1_000, 43_200_000) } - }; + }); // Up to 10 % jitter, derived deterministically from token bytes so // unit tests are reproducible without injecting an RNG. let jitter_pct = (token.len() % 10) as u64; @@ -820,17 +816,14 @@ mod auth_tests { } #[test] - fn compute_refresh_delay_treats_exp_zero_as_non_expiring() { + fn compute_refresh_delay_treats_exp_zero_as_expired() { use base64::Engine as _; let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(r#"{"exp":0}"#); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); let slot: TokenSlot = Arc::new(RwLock::new(bearer)); let delay = compute_refresh_delay(&slot); - assert!( - (6 * 60 * 60..=7 * 60 * 60).contains(&delay.as_secs()), - "non-expiring tokens should use the fallback refresh delay, got {delay:?}" - ); + assert!((1..60).contains(&delay.as_secs())); } #[test] diff --git a/crates/openshell-driver-kubernetes-secrets/src/lib.rs b/crates/openshell-driver-kubernetes-secrets/src/lib.rs index e0250bbfca..4cd5f5ed7e 100644 --- a/crates/openshell-driver-kubernetes-secrets/src/lib.rs +++ b/crates/openshell-driver-kubernetes-secrets/src/lib.rs @@ -48,32 +48,26 @@ impl CredentialDriverService { } } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -enum WorkspaceMode { - #[default] - Shared, - Managed, - Operator, -} - #[derive(Debug, Clone, PartialEq, Eq)] struct KubernetesSecretsDriverSettings { namespace: String, - allow_reference_namespace: bool, - workspace_mode: WorkspaceMode, - gateway_id: String, } impl KubernetesSecretsDriverSettings { - fn target_namespace(&self, workspace: &str) -> String { - match self.workspace_mode { - WorkspaceMode::Shared => self.namespace.clone(), - WorkspaceMode::Managed => { - format!("openshell-{}-{}", self.gateway_id, workspace) - } - WorkspaceMode::Operator => workspace.to_string(), + fn resolve_handle( + &self, + handle: &CredentialHandle, + credential_key: &str, + ) -> Result { + let reference = KubernetesSecretsCredentialDriver::parse_handle(handle, credential_key)?; + if reference.namespace != self.namespace { + return Err(Status::permission_denied(format!( + "kubernetes-secrets credential handle references namespace '{}' but the driver \ + stores credentials in namespace '{}'", + reference.namespace, self.namespace + ))); } + Ok(reference) } } @@ -81,9 +75,6 @@ impl KubernetesSecretsDriverSettings { #[serde(default, deny_unknown_fields)] struct KubernetesSecretsDriverConfig { namespace: Option, - allow_reference_namespace: bool, - workspace_mode: WorkspaceMode, - gateway_id: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -158,21 +149,7 @@ impl KubernetesSecretsCredentialDriver { handle: &CredentialHandle, credential_key: &str, ) -> Result { - let reference = Self::parse_handle(handle, credential_key)?; - // In managed/operator modes secrets live in workspace-specific - // namespaces so cross-namespace handles are expected. - if self.settings.workspace_mode == WorkspaceMode::Shared - && reference.namespace != self.settings.namespace - && !self.settings.allow_reference_namespace - { - return Err(Status::permission_denied(format!( - "kubernetes-secrets credential handle references namespace '{}' but the driver is \ - configured for namespace '{}'; set allow_reference_namespace = true to allow \ - cross-namespace references", - reference.namespace, self.settings.namespace - ))); - } - Ok(reference) + self.settings.resolve_handle(handle, credential_key) } pub async fn store_credential( @@ -203,7 +180,7 @@ impl KubernetesSecretsCredentialDriver { reference } else { KubernetesSecretReference { - namespace: self.settings.target_namespace(&request.workspace), + namespace: self.settings.namespace.clone(), secret_name: managed_secret_name( &request.workspace, &request.provider_id, @@ -546,30 +523,8 @@ impl KubernetesSecretsDriverSettings { } None => default_namespace(), }; - let gateway_id = config.gateway_id.unwrap_or_default(); - if config.workspace_mode == WorkspaceMode::Managed { - let gateway_id = trimmed_config_string("gateway_id", &gateway_id)?; - if !is_dns_label(gateway_id) { - return Err(Error::config( - "[openshell.credential_drivers.kubernetes-secrets] gateway_id must be a DNS-1123 label in managed workspace mode", - )); - } - // Workspace names are limited to 19 characters by the gateway. - // Keep the longest generated namespace within Kubernetes' 63-char - // DNS label limit, matching the Kubernetes compute driver. - if "openshell-".len() + gateway_id.len() + 1 + 19 > 63 { - return Err(Error::config( - "[openshell.credential_drivers.kubernetes-secrets] gateway_id is too long for managed workspace mode", - )); - } - } - Ok(Self { - namespace, - allow_reference_namespace: config.allow_reference_namespace, - workspace_mode: config.workspace_mode, - gateway_id, - }) + Ok(Self { namespace }) } } @@ -810,12 +765,10 @@ mod tests { fn settings_parse_configured_namespace() { let settings = KubernetesSecretsDriverSettings::from_table(&toml::toml! { namespace = "openshell" - allow_reference_namespace = true }) .unwrap(); assert_eq!(settings.namespace, "openshell"); - assert!(settings.allow_reference_namespace); } #[test] @@ -830,57 +783,25 @@ mod tests { } #[test] - fn settings_reject_invalid_namespace() { - let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { - namespace = "OpenShell" - }) - .unwrap_err(); - - assert!(err.to_string().contains("namespace")); - } - - #[test] - fn settings_managed_mode_requires_gateway_id() { - let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { - workspace_mode = "managed" - }) - .unwrap_err(); - - assert!(err.to_string().contains("gateway_id")); - } - - #[test] - fn settings_managed_mode_rejects_invalid_gateway_id() { - let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { - workspace_mode = "managed" - gateway_id = "Invalid_Gateway" - }) - .unwrap_err(); - - assert!(err.to_string().contains("DNS-1123")); + fn settings_reject_removed_placement_fields() { + for table in [ + toml::toml! { workspace_mode = "managed" }, + toml::toml! { gateway_id = "gateway-1" }, + toml::toml! { allow_reference_namespace = true }, + ] { + let err = KubernetesSecretsDriverSettings::from_table(&table).unwrap_err(); + assert!(err.to_string().contains("unknown field")); + } } #[test] - fn settings_managed_mode_rejects_gateway_id_that_is_too_long() { - let gateway_id = "a".repeat(35); + fn settings_reject_invalid_namespace() { let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { - workspace_mode = "managed" - gateway_id = gateway_id + namespace = "OpenShell" }) .unwrap_err(); - assert!(err.to_string().contains("too long")); - } - - #[test] - fn settings_managed_mode_accepts_valid_gateway_id() { - let settings = KubernetesSecretsDriverSettings::from_table(&toml::toml! { - workspace_mode = "managed" - gateway_id = "gateway-1" - }) - .unwrap(); - - assert_eq!(settings.gateway_id, "gateway-1"); + assert!(err.to_string().contains("namespace")); } #[test] @@ -943,62 +864,31 @@ mod tests { } #[test] - fn handle_rejects_cross_namespace_when_not_allowed() { + fn handle_rejects_other_namespace() { let settings = KubernetesSecretsDriverSettings { namespace: "openshell".to_string(), - allow_reference_namespace: false, - workspace_mode: WorkspaceMode::Shared, - gateway_id: String::new(), }; - let reference = KubernetesSecretsCredentialDriver::parse_handle( - &handle("v1:other-namespace:provider-secret"), - "API_KEY", - ) - .unwrap(); - assert_eq!(reference.namespace, "other-namespace"); + let err = settings + .resolve_handle(&handle("v1:team-a:provider-secret"), "API_KEY") + .unwrap_err(); - let result = - if reference.namespace != settings.namespace && !settings.allow_reference_namespace { - Err(Status::permission_denied("cross-namespace")) - } else { - Ok(reference) - }; - assert_eq!(result.unwrap_err().code(), Code::PermissionDenied); + assert_eq!(err.code(), Code::PermissionDenied); + assert!(err.message().contains("team-a")); } #[test] - fn handle_allows_cross_namespace_when_configured() { + fn handle_allows_configured_namespace() { let settings = KubernetesSecretsDriverSettings { namespace: "openshell".to_string(), - allow_reference_namespace: true, - workspace_mode: WorkspaceMode::Shared, - gateway_id: String::new(), }; - let reference = KubernetesSecretsCredentialDriver::parse_handle( - &handle("v1:other-namespace:provider-secret"), - "API_KEY", - ) - .unwrap(); - let result = - if reference.namespace != settings.namespace && !settings.allow_reference_namespace { - Err(Status::permission_denied("cross-namespace")) - } else { - Ok(reference) - }; - assert!(result.is_ok()); - } - - #[test] - fn handle_allows_same_namespace() { - let reference = KubernetesSecretsCredentialDriver::parse_handle( - &handle("v1:openshell:provider-secret"), - "API_KEY", - ) - .unwrap(); + let reference = settings + .resolve_handle(&handle("v1:openshell:provider-secret"), "API_KEY") + .unwrap(); assert_eq!(reference.namespace, "openshell"); + assert_eq!(reference.secret_name, "provider-secret"); } #[test] @@ -1174,40 +1064,4 @@ mod tests { assert_eq!(err.code(), Code::FailedPrecondition); assert!(err.message().contains("is not managed by OpenShell")); } - - #[test] - fn target_namespace_shared_returns_static_namespace() { - let settings = KubernetesSecretsDriverSettings { - namespace: "openshell".to_string(), - allow_reference_namespace: false, - workspace_mode: WorkspaceMode::Shared, - gateway_id: String::new(), - }; - assert_eq!(settings.target_namespace("team-a"), "openshell"); - assert_eq!(settings.target_namespace("team-b"), "openshell"); - } - - #[test] - fn target_namespace_managed_computes_from_workspace() { - let settings = KubernetesSecretsDriverSettings { - namespace: "openshell".to_string(), - allow_reference_namespace: false, - workspace_mode: WorkspaceMode::Managed, - gateway_id: "gw1".to_string(), - }; - assert_eq!(settings.target_namespace("team-a"), "openshell-gw1-team-a"); - assert_eq!(settings.target_namespace("team-b"), "openshell-gw1-team-b"); - } - - #[test] - fn target_namespace_operator_uses_workspace_name() { - let settings = KubernetesSecretsDriverSettings { - namespace: "openshell".to_string(), - allow_reference_namespace: false, - workspace_mode: WorkspaceMode::Operator, - gateway_id: String::new(), - }; - assert_eq!(settings.target_namespace("team-a"), "team-a"); - assert_eq!(settings.target_namespace("prod-ns"), "prod-ns"); - } } diff --git a/crates/openshell-driver-kubernetes-secrets/src/main.rs b/crates/openshell-driver-kubernetes-secrets/src/main.rs index bb3cdacbdd..2c7f13293f 100644 --- a/crates/openshell-driver-kubernetes-secrets/src/main.rs +++ b/crates/openshell-driver-kubernetes-secrets/src/main.rs @@ -31,13 +31,6 @@ struct Args { #[arg(long, env = "OPENSHELL_KUBERNETES_SECRETS_NAMESPACE")] namespace: Option, - - #[arg( - long, - env = "OPENSHELL_KUBERNETES_SECRETS_ALLOW_REFERENCE_NAMESPACE", - default_value_t = false - )] - allow_reference_namespace: bool, } #[tokio::main] @@ -80,12 +73,6 @@ fn driver_config(args: &Args) -> toml::Table { toml::Value::String(namespace.clone()), ); } - if args.allow_reference_namespace { - config.insert( - "allow_reference_namespace".to_string(), - toml::Value::Boolean(true), - ); - } config } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index d21a6c1ff1..084f197fb6 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -6,7 +6,8 @@ Caller driver config is disabled by default. External resource references need administrator-controlled approval labels in every workspace mode, including before restart and scheduling-gate release. GPU devices are temporarily exempt. Image-pull Secrets are operator-selected gateway configuration rather than caller -attachments. Managed workspace copies carry gateway ownership metadata. +attachments. Managed mode stages an immutable copy for each sandbox runtime +generation. See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). The driver uses the Kubernetes API to create, delete, fetch, and watch sandbox diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 087c7a5a41..c9697021ae 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -634,6 +634,20 @@ impl KubernetesComputeConfig { !matches!(self.workspace_mode, WorkspaceMode::Shared) } + /// Where supervisor Pods read the gateway client TLS material. Outside + /// shared mode it is staged into each generation's bootstrap Secret. + #[must_use] + pub fn supervisor_client_tls(&self) -> crate::sandbox_runtime::SupervisorClientTls<'_> { + use crate::sandbox_runtime::SupervisorClientTls; + if self.client_tls_secret_name.is_empty() { + SupervisorClientTls::Disabled + } else if self.is_multi_namespace() { + SupervisorClientTls::Bootstrap + } else { + SupervisorClientTls::Secret(&self.client_tls_secret_name) + } + } + /// Compute the K8s resource name for a sandbox. /// /// - **Shared:** `{workspace}--{name}` (namespace doesn't provide isolation). diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 5239f72898..d5266f0d3a 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -12,10 +12,9 @@ use crate::isolation::{ BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesSandboxRuntimeBoundarySpec, }; use crate::sandbox_runtime::{ - BOUNDARY_CERTIFICATE_PATH, BOUNDARY_CONFIG_PATH, BOUNDARY_PRIVATE_KEY_PATH, - SANDBOX_SECRET_COMPONENT, SUPERVISOR_SECRET_COMPONENT, - SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, boundary_service, - generate_proxy_ca_material, sandbox_bootstrap_secret, + BOUNDARY_CERTIFICATE_PATH, BOUNDARY_CONFIG_PATH, BOUNDARY_PRIVATE_KEY_PATH, ClientTlsMaterial, + SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, SupervisorClientTls, + boundary_service, generate_proxy_ca_material, image_pull_secret_copy, sandbox_bootstrap_secret, sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, supervisor_bootstrap_secret, supervisor_pod, workload_fence, }; @@ -1197,167 +1196,87 @@ impl KubernetesComputeDriver { Ok(()) } - /// Ensure the client TLS Secret exists in `namespace` by copying it from - /// the gateway's Helm release namespace. Idempotent: creates the Secret on - /// first call, updates it on subsequent calls to pick up cert rotations. - /// No-op when `client_tls_secret_name` is empty (TLS disabled). - async fn ensure_tls_secret(&self, namespace: &str) -> Result<(), KubernetesDriverError> { - if self.config.client_tls_secret_name.is_empty() { - return Ok(()); + /// Names the workload and supervisor Pods of a runtime generation use in + /// `imagePullSecrets`. + fn generation_image_pull_secret_names(&self, names: &SandboxRuntimeNames) -> Vec { + if self.config.workspace_mode == WorkspaceMode::Managed { + names.image_pull_secrets(self.config.image_pull_secrets.len()) + } else { + self.config.image_pull_secrets.clone() } + } + async fn read_source_secret(&self, name: &str) -> Result { let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); - let source = match tokio::time::timeout( - KUBE_API_TIMEOUT, - source_api.get(&self.config.client_tls_secret_name), - ) - .await - { - Ok(Ok(s)) => s, - Ok(Err(e)) => { - warn!( - secret = %self.config.client_tls_secret_name, - source_namespace = %self.config.namespace, - error = %e, - "failed to read source TLS secret" - ); - return Err(KubernetesDriverError::from_kube(e)); - } - Err(_) => { - return Err(KubernetesDriverError::Message(format!( - "timeout reading TLS secret {} from {}", - self.config.client_tls_secret_name, self.config.namespace - ))); - } - }; - - let target_api: Api = Api::namespaced(self.client.clone(), namespace); - let copy = Secret { - metadata: ObjectMeta { - name: Some(self.config.client_tls_secret_name.clone()), - namespace: Some(namespace.to_string()), - labels: Some(BTreeMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }, - data: source.data, - type_: source.type_, - ..Default::default() - }; - - match tokio::time::timeout( - KUBE_API_TIMEOUT, - target_api.patch( - &self.config.client_tls_secret_name, - &PatchParams::apply("openshell"), - &Patch::Apply(©), - ), - ) - .await - { - Ok(Ok(_)) => { - info!( - namespace = %namespace, - secret = %self.config.client_tls_secret_name, - "applied TLS secret copy" - ); - } - Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), - Err(_) => { - return Err(KubernetesDriverError::Message(format!( - "timeout applying TLS secret in {namespace}" - ))); + match tokio::time::timeout(KUBE_API_TIMEOUT, source_api.get(name)).await { + Ok(Ok(secret)) => Ok(secret), + Ok(Err(KubeError::Api(error))) if error.code == 404 => { + Err(KubernetesDriverError::Precondition(format!( + "Secret {name} does not exist in source namespace {}", + self.config.namespace + ))) } + Ok(Err(error)) => Err(KubernetesDriverError::from_kube(error)), + Err(_) => Err(KubernetesDriverError::Message(format!( + "timeout reading Secret {name} from {}", + self.config.namespace + ))), } - - Ok(()) } - /// Copy the explicitly configured image-pull Secrets into a managed - /// workspace namespace. Server-side apply refreshes rotated credentials - /// without forcibly taking fields owned by another manager. - async fn ensure_image_pull_secrets( + /// Create immutable copies of the configured image-pull Secrets for one + /// managed-mode runtime generation. An existing Secret with a generation + /// name fails the create rather than being adopted. + async fn create_generation_image_pull_secrets( &self, namespace: &str, - workspace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + owners: Vec, ) -> Result<(), KubernetesDriverError> { - let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); - let target_api: Api = Api::namespaced(self.client.clone(), namespace); - - for secret_name in &self.config.image_pull_secrets { - let source = match tokio::time::timeout(KUBE_API_TIMEOUT, source_api.get(secret_name)) - .await - { - Ok(Ok(secret)) => secret, - Ok(Err(KubeError::Api(error))) if error.code == 404 => { - return Err(KubernetesDriverError::Precondition(format!( - "configured image-pull Secret {secret_name} does not exist in source namespace {}", - self.config.namespace - ))); - } - Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), - Err(_) => { - return Err(KubernetesDriverError::Message(format!( - "timeout reading image-pull Secret {secret_name} from {}", - self.config.namespace - ))); - } - }; - - let existing = tokio::time::timeout(KUBE_API_TIMEOUT, target_api.get_opt(secret_name)) + if self.config.workspace_mode != WorkspaceMode::Managed { + return Ok(()); + } + let secrets = Api::::namespaced(self.client.clone(), namespace); + let targets = names.image_pull_secrets(self.config.image_pull_secrets.len()); + for (source_name, target_name) in self.config.image_pull_secrets.iter().zip(&targets) { + let source = self.read_source_secret(source_name).await?; + let copy = + image_pull_secret_copy(namespace, target_name, sandbox_id, &source, owners.clone()); + secrets + .create(&PostParams::default(), ©) .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timeout checking image-pull Secret {secret_name} in {namespace}" - )) - })? .map_err(KubernetesDriverError::from_kube)?; - if existing.as_ref().is_some_and(|existing| { - !image_pull_secret_owned_by_gateway( - existing.metadata.labels.as_ref(), - &self.config.gateway_id, - workspace, - ) - }) { - return Err(KubernetesDriverError::Precondition(format!( - "image-pull Secret {secret_name} in {namespace} is not owned by this gateway" - ))); - } - - let copy = image_pull_secret_copy( - secret_name, - namespace, - workspace, - &self.config.gateway_id, - source, - ); - match tokio::time::timeout( - KUBE_API_TIMEOUT, - target_api.patch( - secret_name, - &PatchParams::apply("openshell"), - &Patch::Apply(©), - ), - ) - .await - { - Ok(Ok(_)) => { - info!(namespace, secret = %secret_name, "applied image-pull Secret copy"); - } - Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), - Err(_) => { - return Err(KubernetesDriverError::Message(format!( - "timeout applying image-pull Secret {secret_name} in {namespace}" - ))); - } - } } - Ok(()) } + /// Read the gateway client TLS material staged into supervisor bootstrap + /// Secrets outside the sandbox namespace. + async fn read_client_tls_material( + &self, + ) -> Result, KubernetesDriverError> { + if self.config.supervisor_client_tls() != SupervisorClientTls::Bootstrap { + return Ok(None); + } + let name = &self.config.client_tls_secret_name; + let source = self.read_source_secret(name).await?; + let mut data = source.data.unwrap_or_default(); + let mut take = |key: &str| { + data.remove(key).map(|value| value.0).ok_or_else(|| { + KubernetesDriverError::Precondition(format!( + "Secret {name} in {} has no {key}", + self.config.namespace + )) + }) + }; + Ok(Some(ClientTlsMaterial { + ca_certificate: take("ca.crt")?, + certificate: take("tls.crt")?, + private_key: take("tls.key")?, + })) + } + /// Delete the managed namespace and all its contents (managed mode only). /// Called via the `DeleteWorkspace` RPC after workspace deletion. /// Kubernetes cascades namespace deletion to all resources within it. @@ -1847,7 +1766,7 @@ impl KubernetesComputeDriver { )] pub async fn create_sandbox(&self, sandbox: &Sandbox) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); - let result = self.create_sandbox_inner(sandbox).await; + let result = Box::pin(self.create_sandbox_inner(sandbox)).await; span_status.finish(result) } @@ -1879,12 +1798,7 @@ impl KubernetesComputeDriver { let target_namespace = match self.config.workspace_mode { WorkspaceMode::Shared => self.config.namespace.clone(), - WorkspaceMode::Managed => { - let namespace = self.ensure_namespace(workspace).await?; - self.ensure_image_pull_secrets(&namespace, workspace) - .await?; - namespace - } + WorkspaceMode::Managed => self.ensure_namespace(workspace).await?, WorkspaceMode::Operator => { if let Some(ref allowlist) = self.operator_allowlist && !allowlist.contains(workspace) @@ -1897,9 +1811,6 @@ impl KubernetesComputeDriver { } }; - if self.config.is_multi_namespace() { - self.ensure_tls_secret(&target_namespace).await?; - } let resource_identities = self .admit_requested_resources(sandbox) .await @@ -1926,6 +1837,7 @@ impl KubernetesComputeDriver { let generation = random_sandbox_runtime_token(); let proxy_names = SandboxRuntimeNames::for_generation(&sandbox.id, &generation); + let image_pull_secrets = self.generation_image_pull_secret_names(&proxy_names); let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( sandbox.spec.as_ref(), ) @@ -1936,7 +1848,7 @@ impl KubernetesComputeDriver { let params = SandboxPodParams { default_image: &self.config.default_image, image_pull_policy: self.config.image_pull_policy, - image_pull_secrets: &self.config.image_pull_secrets, + image_pull_secrets: &image_pull_secrets, sandbox_runtime_image: &self.config.sandbox_runtime_image, sandbox_runtime_image_pull_policy: self.config.sandbox_runtime_image_pull_policy, service_account_name: &self.config.service_account_name, @@ -2381,6 +2293,7 @@ impl KubernetesComputeDriver { // fails before any Service, Pod, or Secret is created. let upstream_proxy_ca_bundle = read_staged_upstream_proxy_ca_bundle(self.config.proxy_ca_bundle.as_deref()).await?; + let client_tls = self.read_client_tls_material().await?; let cr_uid = sandbox_cr.metadata.uid.as_deref().ok_or_else(|| { KubernetesDriverError::Message("created Sandbox CR has no UID".to_string()) })?; @@ -2442,9 +2355,9 @@ impl KubernetesComputeDriver { &self.config.service_account_name, agent_uid, agent_gid, - &self.config.image_pull_secrets, + &self.generation_image_pull_secret_names(names), &self.config.grpc_endpoint, - &self.config.client_tls_secret_name, + self.config.supervisor_client_tls(), main_process_spec, log_level, self.config.effective_sa_token_ttl_secs(), @@ -2628,6 +2541,8 @@ impl KubernetesComputeDriver { .runtime_descriptor .backend_descriptor() .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let workload_owner = runtime_pod_owner(&workload_pod_name, &workload_pod_uid); + let supervisor_owner = runtime_pod_owner(&names.supervisor_pod, &supervisor_uid); let sandbox_secret = sandbox_bootstrap_secret( namespace, names, @@ -2638,14 +2553,7 @@ impl KubernetesComputeDriver { .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, tls.certificate_chain_pem.into_bytes(), tls.private_key_pem.into_bytes(), - OwnerReference { - api_version: "v1".to_string(), - kind: "Pod".to_string(), - name: workload_pod_name.clone(), - uid: workload_pod_uid, - controller: Some(false), - block_owner_deletion: Some(false), - }, + workload_owner.clone(), ); let supervisor_secret = supervisor_bootstrap_secret( namespace, @@ -2658,14 +2566,8 @@ impl KubernetesComputeDriver { proxy_ca.certificate_pem.into_bytes(), proxy_ca.private_key_pem.into_bytes(), upstream_proxy_ca_bundle.clone(), - OwnerReference { - api_version: "v1".to_string(), - kind: "Pod".to_string(), - name: names.supervisor_pod.clone(), - uid: supervisor_uid.clone(), - controller: Some(false), - block_owner_deletion: Some(false), - }, + client_tls, + supervisor_owner.clone(), ); let secrets = Api::::namespaced(self.client.clone(), namespace); secrets @@ -2676,6 +2578,13 @@ impl KubernetesComputeDriver { .create(&PostParams::default(), &supervisor_secret) .await .map_err(KubernetesDriverError::from_kube)?; + self.create_generation_image_pull_secrets( + namespace, + names, + &sandbox.id, + vec![workload_owner, supervisor_owner], + ) + .await?; pods.patch( &names.supervisor_pod, @@ -2736,6 +2645,7 @@ impl KubernetesComputeDriver { // fails before any Service, Pod, or Secret is created. let upstream_proxy_ca_bundle = read_staged_upstream_proxy_ca_bundle(self.config.proxy_ca_bundle.as_deref()).await?; + let client_tls = self.read_client_tls_material().await?; let namespace_uid = Api::::all(self.client.clone()) .get(namespace) .await @@ -2888,6 +2798,8 @@ impl KubernetesComputeDriver { .runtime_descriptor .backend_descriptor() .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let workload_owner = runtime_pod_owner(&workload_pod_name, &workload_pod_uid); + let supervisor_owner = runtime_pod_owner(&names.supervisor_pod, supervisor_uid); let sandbox_secret = sandbox_bootstrap_secret( namespace, names, @@ -2898,14 +2810,7 @@ impl KubernetesComputeDriver { .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, tls.certificate_chain_pem.into_bytes(), tls.private_key_pem.into_bytes(), - OwnerReference { - api_version: "v1".to_string(), - kind: "Pod".to_string(), - name: workload_pod_name.clone(), - uid: workload_pod_uid, - controller: Some(false), - block_owner_deletion: Some(false), - }, + workload_owner.clone(), ); let supervisor_secret = supervisor_bootstrap_secret( namespace, @@ -2918,14 +2823,8 @@ impl KubernetesComputeDriver { proxy_ca.certificate_pem.into_bytes(), proxy_ca.private_key_pem.into_bytes(), upstream_proxy_ca_bundle.clone(), - OwnerReference { - api_version: "v1".to_string(), - kind: "Pod".to_string(), - name: names.supervisor_pod.clone(), - uid: supervisor_uid.to_string(), - controller: Some(false), - block_owner_deletion: Some(false), - }, + client_tls, + supervisor_owner.clone(), ); let secrets = Api::::namespaced(self.client.clone(), namespace); secrets @@ -2936,6 +2835,13 @@ impl KubernetesComputeDriver { .create(&PostParams::default(), &supervisor_secret) .await .map_err(KubernetesDriverError::from_kube)?; + self.create_generation_image_pull_secrets( + namespace, + names, + sandbox_id, + vec![workload_owner, supervisor_owner], + ) + .await?; pods.patch( &names.supervisor_pod, @@ -3032,8 +2938,12 @@ impl KubernetesComputeDriver { pod_is_gone, ); if stop_is_complete { - self.delete_sandbox_runtime_supervisor(sandbox_id, &namespace) - .await?; + self.delete_sandbox_runtime_supervisor( + sandbox_id, + &namespace, + sandbox_runtime_generation(&object).as_slice(), + ) + .await?; patch_dynamic_object_with_resource_version_retry( &agent_sandbox_api.api, &kube_name, @@ -3216,7 +3126,9 @@ impl KubernetesComputeDriver { let names = SandboxRuntimeNames::for_generation(sandbox_id, generation.as_str()); self.create_sandbox_runtime_fence(&namespace, &names) .await?; - self.delete_sandbox_runtime_supervisor(sandbox_id, &namespace) + let mut stale_generations = sandbox_runtime_generation(&object).as_slice().to_vec(); + stale_generations.push(generation.as_str()); + self.delete_sandbox_runtime_supervisor(sandbox_id, &namespace, &stale_generations) .await?; let main_process_spec = required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC)?; @@ -3242,9 +3154,9 @@ impl KubernetesComputeDriver { &self.config.service_account_name, agent_uid, agent_gid, - &self.config.image_pull_secrets, + &self.generation_image_pull_secret_names(&names), &self.config.grpc_endpoint, - &self.config.client_tls_secret_name, + self.config.supervisor_client_tls(), &main_process_spec, &log_level, self.config.effective_sa_token_ttl_secs(), @@ -3300,6 +3212,7 @@ impl KubernetesComputeDriver { ) })?; sandbox_secret["secretName"] = serde_json::json!(names.sandbox_secret); + let image_pull_secrets = self.generation_image_pull_secret_names(&names); let restart = async { patch_dynamic_object_with_resource_version_retry( &sandbox_api.api, @@ -3316,8 +3229,8 @@ impl KubernetesComputeDriver { ANNOTATION_SANDBOX_RUNTIME_GENERATION: generation.as_str(), ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid.clone(), }); - running_patch["spec"]["podTemplate"]["spec"]["volumes"] = - serde_json::Value::Array(volumes.clone()); + running_patch["spec"]["podTemplate"]["spec"] = + restart_pod_template_spec(&volumes, &image_pull_secrets); running_patch }, ) @@ -3499,40 +3412,34 @@ impl KubernetesComputeDriver { &self, sandbox_id: &str, namespace: &str, + generations: &[&str], ) -> Result<(), KubernetesDriverError> { self.delete_sandbox_runtime_supervisor_pod(sandbox_id, namespace) .await?; - self.delete_sandbox_runtime_generation_secrets(sandbox_id, namespace) + self.delete_sandbox_runtime_generation_secrets(sandbox_id, namespace, generations) .await } + /// Delete the bootstrap Secrets of the given runtime generations by exact + /// name. Garbage collection through their owner Pods removes older + /// generations. async fn delete_sandbox_runtime_generation_secrets( &self, sandbox_id: &str, namespace: &str, + generations: &[&str], ) -> Result<(), KubernetesDriverError> { let secrets = Api::::namespaced(self.client.clone(), namespace); - for component in [SANDBOX_SECRET_COMPONENT, SUPERVISOR_SECRET_COMPONENT] { - let selector = - format!("openshell.ai/sandbox-id={sandbox_id},openshell.ai/component={component}"); - let items = secrets - .list(&ListParams::default().labels(&selector)) - .await - .map_err(KubernetesDriverError::from_kube)?; - for secret in items { - let Some(name) = secret.metadata.name else { - continue; - }; - match secrets - .delete( - &name, - &DeleteParams::default().preconditions(Preconditions { - uid: secret.metadata.uid, - resource_version: None, - }), - ) - .await - { + for generation in generations { + let names = SandboxRuntimeNames::for_generation(sandbox_id, generation); + let mut generation_secrets = if self.config.workspace_mode == WorkspaceMode::Managed { + names.image_pull_secrets(self.config.image_pull_secrets.len()) + } else { + Vec::new() + }; + generation_secrets.splice(0..0, [names.sandbox_secret, names.supervisor_secret]); + for name in &generation_secrets { + match secrets.delete(name, &DeleteParams::default()).await { Ok(_) | Err(KubeError::Api(kube::core::ErrorResponse { code: 404, .. })) => {} Err(error) => return Err(KubernetesDriverError::from_kube(error)), } @@ -3884,7 +3791,12 @@ impl KubernetesComputeDriver { } } match self - .reconcile_sandbox_runtime_supervisor(&sandbox_id, namespace, desired_running) + .reconcile_sandbox_runtime_supervisor( + &sandbox_id, + namespace, + desired_running, + sandbox_runtime_generation(&object).as_slice(), + ) .await { Ok(()) => {} @@ -3938,7 +3850,11 @@ impl KubernetesComputeDriver { } } if let Err(error) = self - .delete_sandbox_runtime_supervisor(sandbox_id, namespace) + .delete_sandbox_runtime_supervisor( + sandbox_id, + namespace, + sandbox_runtime_generation(object).as_slice(), + ) .await { warn!(sandbox_id, %error, "could not finish sandbox-runtime suspension cleanup"); @@ -4208,10 +4124,11 @@ impl KubernetesComputeDriver { sandbox_id: &str, namespace: &str, desired_running: bool, + generations: &[&str], ) -> Result<(), KubernetesDriverError> { if !desired_running { return self - .delete_sandbox_runtime_supervisor(sandbox_id, namespace) + .delete_sandbox_runtime_supervisor(sandbox_id, namespace, generations) .await; } let names = SandboxRuntimeNames::new(sandbox_id); @@ -4866,46 +4783,26 @@ fn managed_ssh_network_policy(namespace: &str, config: &KubernetesComputeConfig) } } -fn image_pull_secret_copy( - secret_name: &str, - namespace: &str, - workspace: &str, - gateway_id: &str, - source: Secret, -) -> Secret { - let mut labels = source.metadata.labels.unwrap_or_default(); - labels.insert( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ); - labels.insert(LABEL_GATEWAY_ID.to_string(), gateway_id.to_string()); - labels.insert(LABEL_SANDBOX_WORKSPACE.to_string(), workspace.to_string()); - Secret { - metadata: ObjectMeta { - name: Some(secret_name.to_string()), - namespace: Some(namespace.to_string()), - labels: Some(labels), - ..Default::default() - }, - data: source.data, - type_: source.type_, - ..Default::default() +fn runtime_pod_owner(name: &str, uid: &str) -> OwnerReference { + OwnerReference { + api_version: "v1".to_string(), + kind: "Pod".to_string(), + name: name.to_string(), + uid: uid.to_string(), + controller: Some(false), + block_owner_deletion: Some(false), } } -fn image_pull_secret_owned_by_gateway( - labels: Option<&BTreeMap>, - gateway_id: &str, - workspace: &str, -) -> bool { - labels.is_some_and(|labels| { - labels.get(LABEL_MANAGED_BY).map(String::as_str) == Some(LABEL_MANAGED_BY_VALUE) - && labels - .get(LABEL_GATEWAY_ID) - .is_none_or(|value| value == gateway_id) - && labels - .get(LABEL_SANDBOX_WORKSPACE) - .is_none_or(|value| value == workspace) +/// Pod template fields replaced when a stopped sandbox starts a new runtime +/// generation. +fn restart_pod_template_spec( + volumes: &[serde_json::Value], + image_pull_secrets: &[String], +) -> serde_json::Value { + serde_json::json!({ + "volumes": volumes, + "imagePullSecrets": image_pull_secret_refs(image_pull_secrets), }) } @@ -8440,6 +8337,7 @@ mod tests { "resourceVersion": "42", "annotations": { SANDBOX_POD_NAME_ANNOTATION: "workload-pod", + ANNOTATION_SANDBOX_RUNTIME_GENERATION: "gen-7", ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::Suspending.as_str() } @@ -8455,12 +8353,6 @@ mod tests { serde_json::json!({"kind": "Status", "status": "Success", "code": 200}), ) }; - let no_secrets = || { - kube_test_response( - http::StatusCode::OK, - serde_json::json!({"apiVersion": "v1", "kind": "SecretList", "items": []}), - ) - }; let steps = Arc::new(std::sync::Mutex::new(VecDeque::from([ ( http::Method::GET, @@ -8504,14 +8396,14 @@ mod tests { kube_test_not_found("pods", "os-supervisor-sandbox-1"), ), ( - http::Method::GET, - "/api/v1/namespaces/openshell/secrets", - no_secrets(), + http::Method::DELETE, + "/api/v1/namespaces/openshell/secrets/os-sandbox-sandbox-1-gen7", + success(), ), ( - http::Method::GET, - "/api/v1/namespaces/openshell/secrets", - no_secrets(), + http::Method::DELETE, + "/api/v1/namespaces/openshell/secrets/os-supervisor-sandbox-1-gen7", + kube_test_not_found("secrets", "os-supervisor-sandbox-1-gen7"), ), ( http::Method::GET, @@ -11045,204 +10937,336 @@ mod tests { ); } - #[test] - fn image_pull_secret_copy_keeps_portable_fields_and_records_ownership() { - let source: Secret = serde_json::from_value(serde_json::json!({ - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": "regcred", - "namespace": "gateway", - "uid": "source-uid", - "resourceVersion": "42", - "labels": { "source-only": "true" }, - "annotations": { "source-only": "true" }, - "finalizers": ["example.test/finalizer"] - }, - "type": "kubernetes.io/dockerconfigjson", - "data": { ".dockerconfigjson": "e30=" } - })) - .unwrap(); + type KubeTestStep = ( + http::Method, + &'static str, + http::Response, + ); + + type ScriptedDriver = ( + KubernetesComputeDriver, + Arc>>, + Arc>>, + ); + + /// Build a driver whose client answers `steps` in order and records each + /// request body. + fn scripted_driver( + config: KubernetesComputeConfig, + steps: Vec, + ) -> ScriptedDriver { + use http_body_util::BodyExt as _; + let steps = Arc::new(std::sync::Mutex::new(VecDeque::from(steps))); + let bodies = Arc::new(std::sync::Mutex::new(Vec::new())); + let service_steps = steps.clone(); + let service_bodies = bodies.clone(); + let service = tower::service_fn(move |request: http::Request| { + let steps = service_steps.clone(); + let bodies = service_bodies.clone(); + async move { + let (method, path, response) = steps + .lock() + .unwrap() + .pop_front() + .expect("unexpected Kubernetes API request"); + assert_eq!(request.method(), method); + assert_eq!(request.uri().path(), path); + let body = request.into_body().collect().await.unwrap().to_bytes(); + if !body.is_empty() { + bodies + .lock() + .unwrap() + .push(serde_json::from_slice(&body).unwrap()); + } + Ok::<_, std::convert::Infallible>(response) + } + }); + let client = Client::new(service, "openshell"); + let driver = KubernetesComputeDriver { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + config, + operator_allowlist: None, + }; + (driver, steps, bodies) + } + + fn pod_owner(name: &str, uid: &str) -> OwnerReference { + OwnerReference { + api_version: "v1".to_string(), + kind: "Pod".to_string(), + name: name.to_string(), + uid: uid.to_string(), + controller: Some(false), + block_owner_deletion: Some(false), + } + } + + fn managed_config(image_pull_secrets: &[&str]) -> KubernetesComputeConfig { + KubernetesComputeConfig { + namespace: "openshell".into(), + gateway_id: "gateway-a".into(), + workspace_mode: WorkspaceMode::Managed, + image_pull_secrets: image_pull_secrets.iter().map(ToString::to_string).collect(), + client_tls_secret_name: "openshell-client-tls".into(), + ..Default::default() + } + } - let copy = image_pull_secret_copy("regcred", "workspace", "team-a", "gateway-a", source); - assert_eq!(copy.metadata.name.as_deref(), Some("regcred")); - assert_eq!(copy.metadata.namespace.as_deref(), Some("workspace")); + #[tokio::test] + async fn managed_pods_reference_generation_image_pull_secrets() { + let names = SandboxRuntimeNames::for_generation("sandbox-1", "gen7"); + let driver = KubernetesComputeDriver::new_for_test(managed_config(&["regcred", "backup"])); assert_eq!( - copy.type_.as_deref(), - Some("kubernetes.io/dockerconfigjson") + driver.generation_image_pull_secret_names(&names), + [ + "os-pull-sandbox-1-gen7-0".to_string(), + "os-pull-sandbox-1-gen7-1".to_string() + ] ); - assert!( - copy.data - .as_ref() - .unwrap() - .contains_key(".dockerconfigjson") + for workspace_mode in [WorkspaceMode::Shared, WorkspaceMode::Operator] { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { + workspace_mode, + ..managed_config(&["regcred"]) + }); + assert_eq!( + driver.generation_image_pull_secret_names(&names), + ["regcred".to_string()] + ); + } + } + + #[test] + fn supervisor_client_tls_is_staged_outside_the_sandbox_namespace() { + let assert_tls = |workspace_mode, name: &str, expected: SupervisorClientTls<'_>| { + let config = KubernetesComputeConfig { + workspace_mode, + client_tls_secret_name: name.into(), + ..Default::default() + }; + assert_eq!(config.supervisor_client_tls(), expected); + }; + assert_tls( + WorkspaceMode::Shared, + "client-tls", + SupervisorClientTls::Secret("client-tls"), ); - assert_eq!( - copy.metadata - .labels - .as_ref() - .unwrap() - .get(LABEL_MANAGED_BY) - .map(String::as_str), - Some(LABEL_MANAGED_BY_VALUE) + assert_tls( + WorkspaceMode::Managed, + "client-tls", + SupervisorClientTls::Bootstrap, ); - assert_eq!( - copy.metadata - .labels - .as_ref() - .unwrap() - .get(LABEL_GATEWAY_ID) - .map(String::as_str), - Some("gateway-a") + assert_tls( + WorkspaceMode::Operator, + "client-tls", + SupervisorClientTls::Bootstrap, ); + assert_tls(WorkspaceMode::Managed, "", SupervisorClientTls::Disabled); + } + + #[test] + fn restart_template_references_the_generation_image_pull_secrets() { + let volumes = vec![serde_json::json!({"name": "bootstrap"})]; + let spec = restart_pod_template_spec(&volumes, &["os-pull-sandbox-1-gen7-0".to_string()]); + assert_eq!(spec["volumes"], serde_json::json!(volumes)); assert_eq!( - copy.metadata - .labels - .as_ref() - .unwrap() - .get(LABEL_SANDBOX_WORKSPACE) - .map(String::as_str), - Some("team-a") + spec["imagePullSecrets"], + serde_json::json!([{"name": "os-pull-sandbox-1-gen7-0"}]) + ); + } + + #[tokio::test] + async fn managed_generation_image_pull_secrets_are_created_without_reading_targets() { + let source = serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "regcred", "namespace": "openshell"}, + "type": "kubernetes.io/dockerconfigjson", + "data": {".dockerconfigjson": "e30="} + }); + let (driver, steps, bodies) = scripted_driver( + managed_config(&["regcred"]), + vec![ + ( + http::Method::GET, + "/api/v1/namespaces/openshell/secrets/regcred", + kube_test_response(http::StatusCode::OK, source), + ), + ( + http::Method::POST, + "/api/v1/namespaces/managed-team-a/secrets", + kube_test_response( + http::StatusCode::CREATED, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "os-pull-sandbox-1-gen7-0", "namespace": "managed-team-a"} + }), + ), + ), + ], ); + let names = SandboxRuntimeNames::for_generation("sandbox-1", "gen7"); + driver + .create_generation_image_pull_secrets( + "managed-team-a", + &names, + "sandbox-1", + vec![ + pod_owner("workload", "w-uid"), + pod_owner("supervisor", "s-uid"), + ], + ) + .await + .expect("generation image-pull Secret should be created"); + assert!(steps.lock().unwrap().is_empty()); + let created = &bodies.lock().unwrap()[0]; + assert_eq!(created["metadata"]["name"], "os-pull-sandbox-1-gen7-0"); + assert_eq!(created["immutable"], true); + assert_eq!(created["type"], "kubernetes.io/dockerconfigjson"); + assert_eq!(created["data"][".dockerconfigjson"], "e30="); assert_eq!( - copy.metadata - .labels - .as_ref() + created["metadata"]["ownerReferences"] + .as_array() .unwrap() - .get("source-only") - .map(String::as_str), - Some("true") + .len(), + 2 ); - assert!(copy.metadata.uid.is_none()); - assert!(copy.metadata.resource_version.is_none()); - assert!(copy.metadata.annotations.is_none()); - assert!(copy.metadata.finalizers.is_none()); } - #[test] - fn image_pull_secret_collision_requires_gateway_ownership() { - let legacy = BTreeMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )]); - assert!(image_pull_secret_owned_by_gateway( - Some(&legacy), - "gateway-a", - "team-a" - )); + #[tokio::test] + async fn generation_image_pull_secret_name_collision_fails_closed() { + let (driver, steps, _) = scripted_driver( + managed_config(&["regcred"]), + vec![ + ( + http::Method::GET, + "/api/v1/namespaces/openshell/secrets/regcred", + kube_test_response( + http::StatusCode::OK, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "regcred", "namespace": "openshell"}, + "type": "kubernetes.io/dockerconfigjson", + "data": {".dockerconfigjson": "e30="} + }), + ), + ), + ( + http::Method::POST, + "/api/v1/namespaces/managed-team-a/secrets", + kube_test_response( + http::StatusCode::CONFLICT, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Status", + "status": "Failure", + "reason": "AlreadyExists", + "message": "secrets \"os-pull-sandbox-1-gen7-0\" already exists", + "code": 409 + }), + ), + ), + ], + ); + let names = SandboxRuntimeNames::for_generation("sandbox-1", "gen7"); + driver + .create_generation_image_pull_secrets("managed-team-a", &names, "sandbox-1", Vec::new()) + .await + .expect_err("an existing Secret must not be adopted"); + assert!(steps.lock().unwrap().is_empty()); + } - let owned = BTreeMap::from([ - ( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ), - (LABEL_GATEWAY_ID.to_string(), "gateway-a".to_string()), - (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), - ]); - assert!(image_pull_secret_owned_by_gateway( - Some(&owned), - "gateway-a", - "team-a" - )); - assert!(!image_pull_secret_owned_by_gateway( - Some(&owned), - "gateway-b", - "team-a" - )); - assert!(!image_pull_secret_owned_by_gateway( - Some(&owned), - "gateway-a", - "team-b" - )); - assert!(!image_pull_secret_owned_by_gateway( - Some(&BTreeMap::new()), - "gateway-a", - "team-a" - )); + #[tokio::test] + async fn unmanaged_modes_create_no_generation_image_pull_secrets() { + for workspace_mode in [WorkspaceMode::Shared, WorkspaceMode::Operator] { + let (driver, _, _) = scripted_driver( + KubernetesComputeConfig { + workspace_mode, + ..managed_config(&["regcred"]) + }, + Vec::new(), + ); + driver + .create_generation_image_pull_secrets( + "workspace", + &SandboxRuntimeNames::for_generation("sandbox-1", "gen7"), + "sandbox-1", + Vec::new(), + ) + .await + .expect("no Secret is staged outside managed mode"); + } } #[tokio::test] - async fn managed_image_pull_secret_copies_operator_selected_source() { - let source_path = "/api/v1/namespaces/openshell/secrets/regcred"; - let target_path = "/api/v1/namespaces/managed-team-a/secrets/regcred"; - let copied = serde_json::json!({ - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": "regcred", - "namespace": "managed-team-a", - "labels": { - "openshell.ai/sandbox-attachable": "true", - LABEL_MANAGED_BY: LABEL_MANAGED_BY_VALUE, - LABEL_GATEWAY_ID: "gateway-a", - LABEL_SANDBOX_WORKSPACE: "team-a" - } - }, - "type": "kubernetes.io/dockerconfigjson", - "data": { ".dockerconfigjson": "e30=" } - }); - let steps = Arc::new(std::sync::Mutex::new(VecDeque::from([ - ( + async fn client_tls_material_is_read_from_the_source_namespace() { + let (driver, steps, _) = scripted_driver( + managed_config(&[]), + vec![( http::Method::GET, - source_path, + "/api/v1/namespaces/openshell/secrets/openshell-client-tls", kube_test_response( http::StatusCode::OK, serde_json::json!({ "apiVersion": "v1", "kind": "Secret", - "metadata": { - "name": "regcred", - "namespace": "openshell", - "labels": {} - }, - "type": "kubernetes.io/dockerconfigjson", - "data": { ".dockerconfigjson": "e30=" } + "metadata": {"name": "openshell-client-tls", "namespace": "openshell"}, + "type": "kubernetes.io/tls", + "data": {"ca.crt": "Y2E=", "tls.crt": "Y2VydA==", "tls.key": "a2V5"} }), ), - ), - ( - http::Method::GET, - target_path, - kube_test_not_found("secrets", "regcred"), - ), - ( - http::Method::PATCH, - target_path, - kube_test_response(http::StatusCode::OK, copied), - ), - ]))); - let service_steps = steps.clone(); - let service = tower::service_fn(move |request: http::Request| { - let steps = service_steps.clone(); - async move { - let (method, path, response) = steps - .lock() - .unwrap() - .pop_front() - .expect("unexpected Kubernetes API request"); - assert_eq!(request.method(), method); - assert_eq!(request.uri().path(), path); - Ok::<_, std::convert::Infallible>(response) - } - }); - let client = Client::new(service, "openshell"); - let driver = KubernetesComputeDriver { - client: client.clone(), - watch_client: client, - sandbox_api_version: Arc::new(OnceCell::new()), - config: KubernetesComputeConfig { - namespace: "openshell".into(), - gateway_id: "gateway-a".into(), - image_pull_secrets: vec!["regcred".into()], - ..Default::default() + )], + ); + let material = driver + .read_client_tls_material() + .await + .expect("read client TLS") + .expect("client TLS is staged in managed mode"); + assert_eq!(material.ca_certificate, b"ca"); + assert_eq!(material.certificate, b"cert"); + assert_eq!(material.private_key, b"key"); + assert!(steps.lock().unwrap().is_empty()); + + let (shared, _, _) = scripted_driver( + KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Shared, + ..managed_config(&[]) }, - operator_allowlist: None, - }; + Vec::new(), + ); + assert!(shared.read_client_tls_material().await.unwrap().is_none()); + } + #[tokio::test] + async fn generation_cleanup_deletes_managed_image_pull_secrets_by_name() { + let success = || kube_test_response(http::StatusCode::OK, serde_json::json!({})); + let (driver, steps, _) = scripted_driver( + managed_config(&["regcred"]), + vec![ + ( + http::Method::DELETE, + "/api/v1/namespaces/managed-team-a/secrets/os-sandbox-sandbox-1-gen7", + success(), + ), + ( + http::Method::DELETE, + "/api/v1/namespaces/managed-team-a/secrets/os-supervisor-sandbox-1-gen7", + success(), + ), + ( + http::Method::DELETE, + "/api/v1/namespaces/managed-team-a/secrets/os-pull-sandbox-1-gen7-0", + kube_test_not_found("secrets", "os-pull-sandbox-1-gen7-0"), + ), + ], + ); driver - .ensure_image_pull_secrets("managed-team-a", "team-a") + .delete_sandbox_runtime_generation_secrets("sandbox-1", "managed-team-a", &["gen7"]) .await - .expect("operator-selected source should be copied"); + .expect("generation Secrets are deleted"); assert!(steps.lock().unwrap().is_empty()); } diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 07c0572436..0588bbb95c 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -25,6 +25,7 @@ use crate::isolation::{ pub const SANDBOX_SECRET_COMPONENT: &str = "sandbox-bootstrap"; pub const SUPERVISOR_SECRET_COMPONENT: &str = "supervisor-bootstrap"; +pub const IMAGE_PULL_SECRET_COMPONENT: &str = "image-pull"; pub const BOUNDARY_CONFIG_KEY: &str = "boundary.json"; pub const BACKEND_DESCRIPTOR_KEY: &str = "runtime-descriptor.json"; pub const BOUNDARY_CERTIFICATE_KEY: &str = "tls.crt"; @@ -36,6 +37,9 @@ pub const PROXY_CA_PRIVATE_KEY: &str = "proxy-ca.key"; /// bundle (`proxy_ca_bundle`). Distinct from [`PROXY_CA_CERTIFICATE_KEY`], /// which is the sandbox's own generated TLS-interception CA. pub const UPSTREAM_PROXY_CA_BUNDLE_KEY: &str = "upstream-proxy-ca.pem"; +pub const CLIENT_TLS_CA_KEY: &str = "client-ca.crt"; +pub const CLIENT_TLS_CERTIFICATE_KEY: &str = "client-tls.crt"; +pub const CLIENT_TLS_PRIVATE_KEY: &str = "client-tls.key"; pub const SANDBOX_BOOTSTRAP_INPUT_PATH: &str = "/.openshell/bootstrap-input"; pub const BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json"; pub const BOUNDARY_CERTIFICATE_PATH: &str = "/.openshell/state/bootstrap/tls.crt"; @@ -50,11 +54,32 @@ pub const PROXY_CA_PRIVATE_KEY_PATH: &str = "/.openshell/supervisor/proxy-ca.key /// `/.openshell/supervisor` with no `items` filter, so the key appears here /// without a dedicated volume or mount. pub const UPSTREAM_PROXY_CA_BUNDLE_PATH: &str = "/.openshell/supervisor/upstream-proxy-ca.pem"; +pub const CLIENT_TLS_CA_PATH: &str = "/.openshell/supervisor/client-ca.crt"; +pub const CLIENT_TLS_CERTIFICATE_PATH: &str = "/.openshell/supervisor/client-tls.crt"; +pub const CLIENT_TLS_PRIVATE_KEY_PATH: &str = "/.openshell/supervisor/client-tls.key"; pub const CONTROL_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock"; pub const NAMESPACE_WORKLOAD_POLICY_NAME: &str = "openshell-sandbox-workloads"; pub const NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME: &str = "openshell-sandbox-supervisors"; pub const SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS: i64 = 30; +/// Gateway client TLS material staged into the supervisor bootstrap Secret. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ClientTlsMaterial { + pub ca_certificate: Vec, + pub certificate: Vec, + pub private_key: Vec, +} + +/// Where the supervisor reads its gateway client TLS material. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SupervisorClientTls<'a> { + Disabled, + /// A Secret in the sandbox namespace, mounted by name. + Secret(&'a str), + /// Keys staged into the supervisor bootstrap Secret. + Bootstrap, +} + pub struct ProxyCaMaterial { pub certificate_pem: String, pub private_key_pem: String, @@ -88,6 +113,7 @@ pub struct SandboxRuntimeNames { pub supervisor_pod: String, pub workload_policy: String, pub supervisor_policy: String, + image_pull_secret_prefix: String, } impl SandboxRuntimeNames { @@ -101,6 +127,7 @@ impl SandboxRuntimeNames { supervisor_pod: format!("os-supervisor-{suffix}"), workload_policy: NAMESPACE_WORKLOAD_POLICY_NAME.to_string(), supervisor_policy: NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME.to_string(), + image_pull_secret_prefix: format!("os-pull-{suffix}"), } } @@ -117,8 +144,18 @@ impl SandboxRuntimeNames { .to_ascii_lowercase(); names.sandbox_secret = format!("{}-{generation}", names.sandbox_secret); names.supervisor_secret = format!("{}-{generation}", names.supervisor_secret); + names.image_pull_secret_prefix = format!("{}-{generation}", names.image_pull_secret_prefix); names } + + /// Names of the image-pull Secrets staged for this generation, one per + /// configured source Secret. + #[must_use] + pub fn image_pull_secrets(&self, count: usize) -> Vec { + (0..count) + .map(|index| format!("{}-{index}", self.image_pull_secret_prefix)) + .collect() + } } #[must_use] @@ -186,7 +223,7 @@ pub fn supervisor_pod( control_gid: u32, image_pull_secrets: &[String], grpc_endpoint: &str, - client_tls_secret_name: &str, + client_tls: SupervisorClientTls<'_>, main_process_spec: &str, log_level: &str, sa_token_ttl_secs: i64, @@ -260,24 +297,32 @@ pub fn supervisor_pod( empty_dir_volume("run"), empty_dir_volume("logs"), ]; - if !client_tls_secret_name.is_empty() { - environment.extend([ - env_var("OPENSHELL_TLS_CA", "/var/run/secrets/openshell-tls/ca.crt"), - env_var( - "OPENSHELL_TLS_CERT", - "/var/run/secrets/openshell-tls/tls.crt", - ), - env_var( - "OPENSHELL_TLS_KEY", - "/var/run/secrets/openshell-tls/tls.key", - ), - ]); - volume_mounts.push(volume_mount( - "client-tls", - "/var/run/secrets/openshell-tls", - true, - )); - volumes.push(secret_volume("client-tls", client_tls_secret_name, None)); + match client_tls { + SupervisorClientTls::Disabled => {} + SupervisorClientTls::Secret(secret_name) => { + environment.extend([ + env_var("OPENSHELL_TLS_CA", "/var/run/secrets/openshell-tls/ca.crt"), + env_var( + "OPENSHELL_TLS_CERT", + "/var/run/secrets/openshell-tls/tls.crt", + ), + env_var( + "OPENSHELL_TLS_KEY", + "/var/run/secrets/openshell-tls/tls.key", + ), + ]); + volume_mounts.push(volume_mount( + "client-tls", + "/var/run/secrets/openshell-tls", + true, + )); + volumes.push(secret_volume("client-tls", secret_name, None)); + } + SupervisorClientTls::Bootstrap => environment.extend([ + env_var("OPENSHELL_TLS_CA", CLIENT_TLS_CA_PATH), + env_var("OPENSHELL_TLS_CERT", CLIENT_TLS_CERTIFICATE_PATH), + env_var("OPENSHELL_TLS_KEY", CLIENT_TLS_PRIVATE_KEY_PATH), + ]), } let mut command = vec![ "/openshell-supervisor".to_string(), @@ -486,6 +531,7 @@ pub fn supervisor_bootstrap_secret( proxy_ca_certificate: Vec, proxy_ca_private_key: Vec, upstream_proxy_ca_bundle: Option>, + client_tls: Option, owner: OwnerReference, ) -> Secret { let mut data = BTreeMap::from([ @@ -511,6 +557,22 @@ pub fn supervisor_bootstrap_secret( if let Some(bundle) = upstream_proxy_ca_bundle { data.insert(UPSTREAM_PROXY_CA_BUNDLE_KEY.to_string(), ByteString(bundle)); } + if let Some(tls) = client_tls { + data.extend([ + ( + CLIENT_TLS_CA_KEY.to_string(), + ByteString(tls.ca_certificate), + ), + ( + CLIENT_TLS_CERTIFICATE_KEY.to_string(), + ByteString(tls.certificate), + ), + ( + CLIENT_TLS_PRIVATE_KEY.to_string(), + ByteString(tls.private_key), + ), + ]); + } Secret { metadata: ObjectMeta { name: Some(names.supervisor_secret.clone()), @@ -526,6 +588,31 @@ pub fn supervisor_bootstrap_secret( } } +/// Copy a configured image-pull Secret into an immutable Secret for one +/// runtime generation. +#[must_use] +pub fn image_pull_secret_copy( + namespace: &str, + name: &str, + sandbox_id: &str, + source: &Secret, + owners: Vec, +) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + owner_references: Some(owners), + labels: Some(common_labels(sandbox_id, IMAGE_PULL_SECRET_COMPONENT)), + ..Default::default() + }, + data: source.data.clone(), + immutable: Some(true), + type_: source.type_.clone(), + ..Default::default() + } +} + #[must_use] pub fn sandbox_owner_reference( name: &str, @@ -637,6 +724,171 @@ mod tests { ); } + #[test] + fn generation_names_include_per_generation_image_pull_secrets() { + let first = SandboxRuntimeNames::for_generation("pair", "ABCdef123456789"); + let second = SandboxRuntimeNames::for_generation("pair", "zyx987654321000"); + assert_eq!( + first.image_pull_secrets(2), + ["os-pull-pair-abcdef123456-0", "os-pull-pair-abcdef123456-1"] + ); + assert!(second.image_pull_secrets(1)[0] != first.image_pull_secrets(1)[0]); + assert!(first.image_pull_secrets(0).is_empty()); + } + + #[test] + fn image_pull_secret_copy_is_immutable_and_owned_by_both_pods() { + let names = SandboxRuntimeNames::for_generation("pair", "gen7"); + let source = Secret { + metadata: ObjectMeta { + name: Some("regcred".to_string()), + namespace: Some("openshell".to_string()), + labels: Some(BTreeMap::from([("team".to_string(), "a".to_string())])), + uid: Some("source-uid".to_string()), + resource_version: Some("42".to_string()), + ..Default::default() + }, + type_: Some("kubernetes.io/dockerconfigjson".to_string()), + data: Some(BTreeMap::from([( + ".dockerconfigjson".to_string(), + ByteString(b"{}".to_vec()), + )])), + ..Default::default() + }; + let owners = vec![owner(), owner()]; + let copy = image_pull_secret_copy( + "workspace", + &names.image_pull_secrets(1)[0], + "pair", + &source, + owners.clone(), + ); + assert_eq!(copy.metadata.name.as_deref(), Some("os-pull-pair-gen7-0")); + assert_eq!(copy.metadata.namespace.as_deref(), Some("workspace")); + assert_eq!(copy.metadata.owner_references, Some(owners)); + assert_eq!( + copy.metadata.labels.as_ref().unwrap()["openshell.ai/component"], + IMAGE_PULL_SECRET_COMPONENT + ); + assert!(copy.metadata.uid.is_none()); + assert!(copy.metadata.resource_version.is_none()); + assert_eq!(copy.immutable, Some(true)); + assert_eq!(copy.type_, source.type_); + assert_eq!(copy.data, source.data); + } + + #[test] + fn supervisor_bootstrap_secret_carries_client_tls_material() { + let names = SandboxRuntimeNames::new("pair"); + let secret = supervisor_bootstrap_secret( + "sandbox", + &names, + "pair", + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + None, + Some(ClientTlsMaterial { + ca_certificate: b"ca".to_vec(), + certificate: b"cert".to_vec(), + private_key: b"key".to_vec(), + }), + owner(), + ); + let data = secret.data.expect("Secret data"); + assert_eq!(data[CLIENT_TLS_CA_KEY].0, b"ca"); + assert_eq!(data[CLIENT_TLS_CERTIFICATE_KEY].0, b"cert"); + assert_eq!(data[CLIENT_TLS_PRIVATE_KEY].0, b"key"); + } + + fn supervisor_pod_with_client_tls(client_tls: SupervisorClientTls<'_>) -> Pod { + supervisor_pod( + "sandbox", + &SandboxRuntimeNames::new("pair"), + "pair", + "demo", + "gateway", + "supervisor:latest", + None, + "sandbox-sa", + 1000, + 1000, + &[], + "https://gateway:8080", + client_tls, + "{}", + "info", + 600, + None, + None, + None, + false, + false, + false, + None, + owner(), + ) + .expect("render supervisor Pod") + } + + fn tls_env_and_volumes(pod: &Pod) -> (BTreeMap, Vec) { + let spec = pod.spec.as_ref().expect("Pod spec"); + let env = spec.containers[0] + .env + .as_ref() + .expect("env") + .iter() + .filter(|variable| variable.name.starts_with("OPENSHELL_TLS_")) + .map(|variable| { + ( + variable.name.clone(), + variable.value.clone().unwrap_or_default(), + ) + }) + .collect(); + let volumes = spec + .volumes + .as_ref() + .expect("volumes") + .iter() + .map(|volume| volume.name.clone()) + .collect(); + (env, volumes) + } + + #[test] + fn supervisor_pod_reads_client_tls_from_the_bootstrap_secret() { + let (env, volumes) = tls_env_and_volumes(&supervisor_pod_with_client_tls( + SupervisorClientTls::Bootstrap, + )); + assert_eq!(env["OPENSHELL_TLS_CA"], CLIENT_TLS_CA_PATH); + assert_eq!(env["OPENSHELL_TLS_CERT"], CLIENT_TLS_CERTIFICATE_PATH); + assert_eq!(env["OPENSHELL_TLS_KEY"], CLIENT_TLS_PRIVATE_KEY_PATH); + assert!(CLIENT_TLS_CA_PATH.starts_with("/.openshell/supervisor/")); + assert!(!volumes.contains(&"client-tls".to_string())); + } + + #[test] + fn supervisor_pod_mounts_a_client_tls_secret_by_name() { + let pod = supervisor_pod_with_client_tls(SupervisorClientTls::Secret("client-tls")); + let (env, volumes) = tls_env_and_volumes(&pod); + assert_eq!( + env["OPENSHELL_TLS_CA"], + "/var/run/secrets/openshell-tls/ca.crt" + ); + assert!(volumes.contains(&"client-tls".to_string())); + } + + #[test] + fn supervisor_pod_omits_client_tls_when_disabled() { + let (env, volumes) = tls_env_and_volumes(&supervisor_pod_with_client_tls( + SupervisorClientTls::Disabled, + )); + assert!(env.is_empty()); + assert!(!volumes.contains(&"client-tls".to_string())); + } + #[test] fn owner_reference_does_not_require_finalizer_mutation_permission() { assert_eq!(owner().block_owner_deletion, Some(false)); @@ -658,7 +910,7 @@ mod tests { 1000, &["registry-credentials".to_string()], "https://gateway:8080", - "client-tls", + SupervisorClientTls::Secret("client-tls"), "{}", "info", 600, @@ -820,6 +1072,7 @@ mod tests { Vec::new(), Vec::new(), None, + None, owner(), ); assert_eq!( @@ -854,6 +1107,7 @@ mod tests { Vec::new(), Vec::new(), Some(b"-----BEGIN CERTIFICATE-----\n".to_vec()), + None, owner(), ); assert_eq!(staged.immutable, Some(true)); @@ -886,7 +1140,7 @@ mod tests { 1000, &[], "https://gateway:8080", - "client-tls", + SupervisorClientTls::Secret("client-tls"), "{}", "info", 600, diff --git a/crates/openshell-isolation-interface/src/contract.rs b/crates/openshell-isolation-interface/src/contract.rs index cc559f5a75..da40e92cb5 100644 --- a/crates/openshell-isolation-interface/src/contract.rs +++ b/crates/openshell-isolation-interface/src/contract.rs @@ -848,22 +848,32 @@ pub trait BoundaryLoopbackConnector: Send + Sync { // Mediation and binary identity // ============================================================================ +/// Path and digest evidence for one executable in a process identity chain. +/// +/// Every instance is authorization-capable, whether it describes the process +/// that owns a connection or one of its executable ancestors. A missing digest +/// is `None`, never an empty value; binary-scoped policy cannot authorize an +/// executable whose digest is unavailable. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExecutableIdentity { + /// Absolute executable path in the workload filesystem namespace. + pub path: PathBuf, + /// Digest of the resolved executable object. `None` when unavailable. + pub digest: Option, +} + /// Executable identity for one accepted connection, resolved by the backend and /// delivered on [`PendingTcpOpen`] before the mediation service evaluates /// policy. /// -/// A missing digest is `None`, never an empty value; policy that requires an -/// unavailable identity field cannot authorize the connection. How a backend -/// resolves identity is private to that backend; the shape and the fail-closed -/// semantics do not change. +/// How a backend resolves identity is private to that backend; the shape and +/// fail-closed semantics do not change. #[derive(Debug, Clone)] pub struct BinaryIdentity { - /// Absolute path of the executable resolved for the accepted connection. - pub binary_path: PathBuf, - /// Digest of the resolved executable object. `None` when unavailable. - pub binary_digest: Option, - /// Ancestor process binaries, nearest first. - pub ancestors: Vec, + /// Executable that owns the accepted connection. + pub executable: ExecutableIdentity, + /// Ancestor process executables, nearest first. + pub ancestors: Vec, /// Absolute script/interpreter paths drawn from the process cmdlines. /// Diagnostic context; never authorizes. pub cmdline_paths: Vec, diff --git a/crates/openshell-isolation-interface/tests/backend_conformance.rs b/crates/openshell-isolation-interface/tests/backend_conformance.rs index 09166e01e1..f40cdcd45a 100644 --- a/crates/openshell-isolation-interface/tests/backend_conformance.rs +++ b/crates/openshell-isolation-interface/tests/backend_conformance.rs @@ -97,9 +97,10 @@ impl NetworkMediationSource for MockSource { Ok(PendingTcpOpen { stream: Box::new(near), binary_identity: Ok(BinaryIdentity { - binary_path: PathBuf::from("/usr/bin/agent"), - binary_digest: K::HAS_DIGEST - .then(|| "00".repeat(32).parse().expect("valid digest")), + executable: ExecutableIdentity { + path: PathBuf::from("/usr/bin/agent"), + digest: K::HAS_DIGEST.then(|| "00".repeat(32).parse().expect("valid digest")), + }, ancestors: vec![], cmdline_paths: vec![], }), @@ -619,7 +620,7 @@ async fn runtime_interfaces_survive_lifecycle_consumption() { let conn = source.accept_tcp().await.expect("accept after consumption"); let identity = conn.binary_identity.expect("identity resolves"); - assert_eq!(identity.binary_path, PathBuf::from("/usr/bin/agent")); + assert_eq!(identity.executable.path, PathBuf::from("/usr/bin/agent")); } // --------------------------------------------------------------------------- @@ -786,10 +787,10 @@ async fn pending_network_open_carries_socket_bound_identity() { .await .expect("accept"); let identity = conn.binary_identity.expect("identity resolves"); - assert_eq!(identity.binary_path, PathBuf::from("/usr/bin/agent")); + assert_eq!(identity.executable.path, PathBuf::from("/usr/bin/agent")); // A missing digest is `None`, never an empty value. assert_eq!( - identity.binary_digest.expect("digest").to_string(), + identity.executable.digest.expect("digest").to_string(), "00".repeat(32) ); assert_eq!(conn.destination, "203.0.113.10:443".parse().unwrap()); @@ -803,7 +804,7 @@ async fn missing_digest_is_none_never_empty() { let source = MockSource::(PhantomData); let conn = source.accept_tcp().await.expect("accept"); let identity = conn.binary_identity.expect("identity resolves"); - assert!(identity.binary_digest.is_none()); + assert!(identity.executable.digest.is_none()); } #[test] diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index ba82e39726..cb83bcf090 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -24,8 +24,8 @@ use openshell_isolation_interface::AgentSpec; use openshell_isolation_interface::contract::Sha256Digest; use openshell_isolation_interface::contract::{ BackendDescriptor, BackendError, BinaryIdentity, BoundaryConfirmation, BoundaryExitStatus, - BoundaryProperties, BoundarySignal, EnforcedProperty, ExecSpec, OuterFenceGuarantees, - ResolveError, ShellSpec, + BoundaryProperties, BoundarySignal, EnforcedProperty, ExecSpec, ExecutableIdentity, + OuterFenceGuarantees, ResolveError, ShellSpec, }; use rcgen::{CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose}; use serde::de::DeserializeOwned; @@ -43,6 +43,20 @@ pub const STREAM_STDIN_CLOSED: u8 = 4; pub const STREAM_NETWORK_DECISION: u8 = 5; pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; +// Every relay, exec, and control exchange shares one HTTP/2 connection. The +// connection window must exceed what all streams can hold unread, otherwise +// stalled relays starve DNS and control traffic of connection-level credit. +// h2 keeps at least two thirds of `connection - in-flight` advertised, so the +// reserve stays usable even with every stream stalled at its window. +pub const BOUNDARY_MAX_CONCURRENT_STREAMS: u32 = 128; +pub const BOUNDARY_STREAM_WINDOW_BYTES: u32 = 256 * 1024; +pub const BOUNDARY_CONNECTION_WINDOW_RESERVE_BYTES: u32 = 16 * 1024 * 1024; +pub const BOUNDARY_CONNECTION_WINDOW_BYTES: u32 = BOUNDARY_MAX_CONCURRENT_STREAMS + * BOUNDARY_STREAM_WINDOW_BYTES + + BOUNDARY_CONNECTION_WINDOW_RESERVE_BYTES; +// HTTP/2 caps any flow-control window at 2^31 - 1. +const _: () = assert!(BOUNDARY_CONNECTION_WINDOW_BYTES <= i32::MAX as u32); + /// Capability masks measured from `/proc//status` by the `OpenShell` /// co-located runtime. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -954,13 +968,37 @@ pub enum BoundaryErrorKind { Process, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutableIdentityWire { + pub path: PathBuf, + pub digest: Option, +} + +impl From for ExecutableIdentityWire { + fn from(identity: ExecutableIdentity) -> Self { + Self { + path: identity.path, + digest: identity.digest, + } + } +} + +impl From for ExecutableIdentity { + fn from(identity: ExecutableIdentityWire) -> Self { + Self { + path: identity.path, + digest: identity.digest, + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "result", rename_all = "snake_case", deny_unknown_fields)] pub enum BinaryIdentityWire { Resolved { - binary_path: PathBuf, - binary_digest: Option, - ancestors: Vec, + executable: ExecutableIdentityWire, + ancestors: Vec, cmdline_paths: Vec, }, Failed { @@ -972,9 +1010,8 @@ impl From> for BinaryIdentityWire { fn from(identity: Result) -> Self { match identity { Ok(identity) => Self::Resolved { - binary_path: identity.binary_path, - binary_digest: identity.binary_digest, - ancestors: identity.ancestors, + executable: identity.executable.into(), + ancestors: identity.ancestors.into_iter().map(Into::into).collect(), cmdline_paths: identity.cmdline_paths, }, Err(error) => Self::Failed { @@ -988,14 +1025,12 @@ impl BinaryIdentityWire { pub fn into_result(self) -> Result { match self { Self::Resolved { - binary_path, - binary_digest, + executable, ancestors, cmdline_paths, } => Ok(BinaryIdentity { - binary_path, - binary_digest, - ancestors, + executable: executable.into(), + ancestors: ancestors.into_iter().map(Into::into).collect(), cmdline_paths, }), Self::Failed { message } => Err(ResolveError::Failed(message)), @@ -1460,15 +1495,21 @@ mod tests { fn binary_identity_wire_rejects_ambiguous_or_invalid_shapes() { for encoded in [ r#"{"result":"resolved","ancestors":[],"cmdline_paths":[]}"#, - r#"{"result":"resolved","binary_path":"/bin/tool","binary_digest":"invalid","ancestors":[],"cmdline_paths":[]}"#, + r#"{"result":"resolved","executable":{"path":"/bin/tool","digest":"invalid"},"ancestors":[],"cmdline_paths":[]}"#, + r#"{"result":"resolved","binary_path":"/bin/tool","binary_digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ancestors":[],"cmdline_paths":[]}"#, r#"{"result":"failed","message":"unavailable","binary_path":"/bin/tool"}"#, ] { assert!(serde_json::from_str::(encoded).is_err()); } let identity = BinaryIdentityWire::from(Ok(BinaryIdentity { - binary_path: PathBuf::from("/bin/tool"), - binary_digest: Some("a".repeat(64).parse().unwrap()), - ancestors: Vec::new(), + executable: ExecutableIdentity { + path: PathBuf::from("/bin/tool"), + digest: Some("a".repeat(64).parse().unwrap()), + }, + ancestors: vec![ExecutableIdentity { + path: PathBuf::from("/bin/launcher"), + digest: Some("b".repeat(64).parse().unwrap()), + }], cmdline_paths: Vec::new(), })); let encoded = serde_json::to_vec(&identity).unwrap(); diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index a6854d52c2..3ed8ed6533 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -52,6 +52,16 @@ const ATTACH_REQUEST_TIMEOUT: Duration = Duration::from_mins(5); /// callers retry whole calls above this; past boot, exhausting this window /// means the remote boundary (or its launcher) is gone rather than still starting. const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +/// Bounds the liveness check that decides whether a stream failure lost the +/// whole connection. +const CONNECTION_PROBE_TIMEOUT: Duration = Duration::from_secs(5); +/// Relay and pending-accept streams share the boundary's concurrent stream +/// limit with exec and control exchanges; leave the remainder for those. +const MAX_NETWORK_STREAMS: usize = 96; +const _: () = assert!( + MAX_NETWORK_STREAMS < crate::boundary_protocol::BOUNDARY_MAX_CONCURRENT_STREAMS as usize +); +const NETWORK_STREAM_WAIT_WARNING: Duration = Duration::from_secs(5); fn begin_recovery_window( deadline: &mut Option, @@ -828,6 +838,8 @@ impl NetworkMediationSource for RemoteNetworkMediation { // An idle accept has no deadline. Recover transport loss before exposing // a source error, which permanently stops the proxy. The boundary drops // the interrupted pending open; a retry never replays its decision. + // Pending accepts and live relays each hold a stream until the relay ends. + let permit = self.client.acquire_network_stream().await; let (stream, response) = self.client.call_wait_stream(Request::AcceptNetwork).await?; let Response::NetworkConnected { identity, @@ -841,7 +853,10 @@ impl NetworkMediationSource for RemoteNetworkMediation { }; let (decision, completion) = tokio::sync::oneshot::channel(); let (proxy_stream, transport_stream) = tokio::io::duplex(64 * 1024); - tokio::spawn(complete_network_open(stream, transport_stream, completion)); + tokio::spawn(async move { + complete_network_open(stream, transport_stream, completion).await; + drop(permit); + }); Ok(PendingTcpOpen { stream: Box::new(proxy_stream), binary_identity: identity.into_result(), @@ -1042,6 +1057,8 @@ struct BoundaryClient { reconnect: tokio::sync::Mutex<()>, next_connection_generation: AtomicU64, credential_monitor_started: AtomicBool, + network_streams: Arc, + network_streams_exhausted: AtomicBool, } #[derive(Clone)] @@ -1050,6 +1067,108 @@ struct CachedGrpcChannel { bearer_fingerprint: [u8; 32], generation: u64, channel: tonic::transport::Channel, + transport: Arc, +} + +/// Closes a channel's physical transport so the boundary observes the +/// disconnect even while abandoned streams still hold channel clones. +#[derive(Default)] +struct TransportAbort { + connected: AtomicBool, + aborted: AtomicBool, + waker: std::sync::Mutex>, +} + +impl TransportAbort { + fn abort(&self) { + self.aborted.store(true, Ordering::Release); + let waker = self + .waker + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if let Some(waker) = waker { + waker.wake(); + } + } + + fn is_aborted(&self) -> bool { + self.aborted.load(Ordering::Acquire) + } + + /// Register before checking so an abort between the two still wakes us. + fn poll_aborted(&self, context: &std::task::Context<'_>) -> bool { + { + let mut waker = self + .waker + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !waker + .as_ref() + .is_some_and(|waker| waker.will_wake(context.waker())) + { + *waker = Some(context.waker().clone()); + } + } + self.is_aborted() + } +} + +struct AbortableTransport { + inner: BoundaryDuplexStream, + abort: Arc, +} + +impl AbortableTransport { + fn aborted_error() -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::ConnectionAborted, + "boundary transport replaced", + ) + } +} + +impl tokio::io::AsyncRead for AbortableTransport { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + context: &mut std::task::Context<'_>, + buffer: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + if self.abort.poll_aborted(context) { + return std::task::Poll::Ready(Err(Self::aborted_error())); + } + std::pin::Pin::new(&mut self.inner).poll_read(context, buffer) + } +} + +impl tokio::io::AsyncWrite for AbortableTransport { + fn poll_write( + mut self: std::pin::Pin<&mut Self>, + context: &mut std::task::Context<'_>, + buffer: &[u8], + ) -> std::task::Poll> { + if self.abort.poll_aborted(context) { + return std::task::Poll::Ready(Err(Self::aborted_error())); + } + std::pin::Pin::new(&mut self.inner).poll_write(context, buffer) + } + + fn poll_flush( + mut self: std::pin::Pin<&mut Self>, + context: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + if self.abort.poll_aborted(context) { + return std::task::Poll::Ready(Err(Self::aborted_error())); + } + std::pin::Pin::new(&mut self.inner).poll_flush(context) + } + + fn poll_shutdown( + mut self: std::pin::Pin<&mut Self>, + context: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.inner).poll_shutdown(context) + } } // Cache only a digest; authenticate with the exact captured header so a @@ -1101,9 +1220,33 @@ impl BoundaryClient { reconnect: tokio::sync::Mutex::new(()), next_connection_generation: AtomicU64::new(1), credential_monitor_started: AtomicBool::new(false), + network_streams: Arc::new(tokio::sync::Semaphore::new(MAX_NETWORK_STREAMS)), + network_streams_exhausted: AtomicBool::new(false), } } + async fn acquire_network_stream(&self) -> tokio::sync::OwnedSemaphorePermit { + let acquire = self.network_streams.clone().acquire_owned(); + tokio::pin!(acquire); + // Bursty clients briefly exceed the budget; only report sustained waits. + let permit = if let Ok(permit) = + tokio::time::timeout(NETWORK_STREAM_WAIT_WARNING, &mut acquire).await + { + self.network_streams_exhausted + .store(false, Ordering::Relaxed); + permit + } else { + if !self.network_streams_exhausted.swap(true, Ordering::Relaxed) { + tracing::warn!( + limit = MAX_NETWORK_STREAMS, + "Sandbox network relay streams exhausted; new sandbox TCP connections are waiting for a relay to close" + ); + } + acquire.await + }; + permit.expect("network stream semaphore is never closed") + } + async fn call_idempotent(&self, request: Request) -> Result { let remember_attach = matches!(request, Request::Attach { .. }); let remember_confirm = matches!(request, Request::Confirm); @@ -1387,10 +1530,8 @@ impl BoundaryClient { "cannot rotate Sandbox Protocol connection before attach".to_string(), ) })?; - let channel = self.build_grpc_channel().await?; - self.exchange_on_channel(channel.clone(), &attach, &credential) - .await?; - self.exchange_on_channel(channel.clone(), &confirm, &credential) + let (channel, transport) = self + .attach_new_channel(&attach, Some(&confirm), &credential) .await?; *self.grpc_channel.lock().await = Some(CachedGrpcChannel { credential_epoch: credential.epoch, @@ -1399,6 +1540,7 @@ impl BoundaryClient { .next_connection_generation .fetch_add(1, Ordering::Relaxed), channel, + transport, }); *self.mediation.lock().await = None; Ok(()) @@ -1420,18 +1562,27 @@ impl BoundaryClient { observed_generation: Option, ) -> Result<(), BackendError> { let _reconnect = self.reconnect.lock().await; - if self - .grpc_channel - .lock() - .await - .as_ref() - .map(|cached| cached.generation) - != observed_generation - { + let cached = self.grpc_channel.lock().await.clone(); + if cached.as_ref().map(|cached| cached.generation) != observed_generation { return Ok(()); } + let confirm = self + .confirm_request + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if let Some(cached) = &cached { + // One failed stream does not prove the connection is gone. Keep a + // live connection: the boundary rejects a same-epoch replacement + // while it still considers the current connection active. + if let Some(confirm) = &confirm + && self.connection_is_live(cached, confirm).await + { + return Ok(()); + } + cached.transport.abort(); + } - *self.grpc_channel.lock().await = None; *self.mediation.lock().await = None; let attach = self .attach_request @@ -1439,21 +1590,30 @@ impl BoundaryClient { .unwrap_or_else(std::sync::PoisonError::into_inner) .clone(); let Some(attach) = attach else { + *self.grpc_channel.lock().await = None; return Ok(()); }; - let confirm = self - .confirm_request - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone(); + // Keep the aborted channel cached until the confirmed replacement is + // swapped in. Concurrent callers then fail fast and wait on this + // recovery instead of caching a connection that never attached. let credential = BoundaryCredential::capture(&self.sandbox_bearer)?; - let channel = self.build_grpc_channel().await?; - self.exchange_on_channel(channel.clone(), &attach, &credential) - .await?; - if let Some(confirm) = confirm { - self.exchange_on_channel(channel.clone(), &confirm, &credential) - .await?; - } + let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; + let (channel, transport) = loop { + match self + .attach_new_channel(&attach, confirm.as_ref(), &credential) + .await + { + Ok(attached) => break attached, + // The boundary may not have retired the aborted connection yet. + Err(BackendError::Unavailable(message)) + if tokio::time::Instant::now() < deadline => + { + tracing::debug!(%message, "boundary reattach not accepted yet; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(error) => return Err(error), + } + }; *self.grpc_channel.lock().await = Some(CachedGrpcChannel { credential_epoch: credential.epoch, bearer_fingerprint: credential.fingerprint, @@ -1461,10 +1621,55 @@ impl BoundaryClient { .next_connection_generation .fetch_add(1, Ordering::Relaxed), channel, + transport, }); Ok(()) } + async fn connection_is_live( + &self, + cached: &CachedGrpcChannel, + confirm: &RequestEnvelope, + ) -> bool { + let Ok(credential) = BoundaryCredential::capture(&self.sandbox_bearer) else { + return false; + }; + matches!( + tokio::time::timeout( + CONNECTION_PROBE_TIMEOUT, + self.exchange_on_channel(cached.channel.clone(), confirm, &credential), + ) + .await, + Ok(Ok(Response::Confirmed { .. })) + ) + } + + /// Open a new physical connection and replay the authenticated lifecycle, + /// closing the connection again if the boundary does not accept it. + async fn attach_new_channel( + &self, + attach: &RequestEnvelope, + confirm: Option<&RequestEnvelope>, + credential: &BoundaryCredential, + ) -> Result<(tonic::transport::Channel, Arc), BackendError> { + let (channel, transport) = self.build_grpc_channel().await?; + let result = async { + self.exchange_on_channel(channel.clone(), attach, credential) + .await?; + if let Some(confirm) = confirm { + self.exchange_on_channel(channel.clone(), confirm, credential) + .await?; + } + Ok(()) + } + .await; + if let Err(error) = result { + transport.abort(); + return Err(error); + } + Ok((channel, transport)) + } + fn start_credential_monitor(self: &Arc) { if self.credential_monitor_started.swap(true, Ordering::AcqRel) { return; @@ -1605,7 +1810,7 @@ impl BoundaryClient { return Ok(cached.channel.clone()); } let credential = BoundaryCredential::capture(&self.sandbox_bearer)?; - let channel = self.build_grpc_channel().await?; + let (channel, transport) = self.build_grpc_channel().await?; *state = Some(CachedGrpcChannel { credential_epoch: credential.epoch, bearer_fingerprint: credential.fingerprint, @@ -1613,25 +1818,38 @@ impl BoundaryClient { .next_connection_generation .fetch_add(1, Ordering::Relaxed), channel: channel.clone(), + transport, }); Ok(channel) } - async fn build_grpc_channel(&self) -> Result { + async fn build_grpc_channel( + &self, + ) -> Result<(tonic::transport::Channel, Arc), BackendError> { let runtime_descriptor = self.runtime_descriptor.clone(); + let transport = Arc::new(TransportAbort::default()); + let connector_transport = transport.clone(); let endpoint = tonic::transport::Endpoint::from_static("http://boundary.openshell.internal") - .initial_stream_window_size(16 * 1024 * 1024) - .initial_connection_window_size(16 * 1024 * 1024) + .initial_stream_window_size(crate::boundary_protocol::BOUNDARY_STREAM_WINDOW_BYTES) + .initial_connection_window_size( + crate::boundary_protocol::BOUNDARY_CONNECTION_WINDOW_BYTES, + ) .http2_keep_alive_interval(Duration::from_secs(10)) .keep_alive_while_idle(true); let channel = endpoint .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { let runtime_descriptor = runtime_descriptor.clone(); + let abort = connector_transport.clone(); async move { + // A channel is one authenticated connection. Never let it + // silently reconnect without replaying attach and confirm. + if abort.is_aborted() || abort.connected.swap(true, Ordering::AcqRel) { + return Err(AbortableTransport::aborted_error()); + } connect_boundary_with_retry(&runtime_descriptor) .await - .map(TokioIo::new) + .map(|inner| TokioIo::new(AbortableTransport { inner, abort })) .map_err(|error| std::io::Error::other(error.to_string())) } })) @@ -1639,7 +1857,7 @@ impl BoundaryClient { .map_err(|error| { BackendError::Unavailable(format!("start boundary gRPC channel: {error}")) })?; - Ok(channel) + Ok((channel, transport)) } #[cfg(test)] @@ -1926,6 +2144,7 @@ fn is_transport_unavailable(message: &str) -> bool { #[cfg(test)] mod tests { mod credential_renewal; + mod flow_control; mod network_recovery; use std::path::PathBuf; @@ -2185,6 +2404,7 @@ mod tests { .fingerprint, generation: 1, channel, + transport: Arc::default(), }); // A caller may try again later, but each call makes exactly one // bounded attach attempt and preserves the server's typed denial. @@ -2255,7 +2475,24 @@ mod tests { Response::Confirmed { .. } )); + // A stream-level failure on a live connection keeps that connection. let failed_generation = client.connection_generation().await; + client + .recover_after_unavailable(failed_generation) + .await + .unwrap(); + assert_eq!(client.connection_generation().await, failed_generation); + assert_eq!(accepted.load(Ordering::Acquire), 1); + assert_eq!(requests.load(Ordering::Acquire), 3); + + client + .grpc_channel + .lock() + .await + .as_ref() + .expect("cached channel") + .transport + .abort(); client .recover_after_unavailable(failed_generation) .await @@ -2265,7 +2502,7 @@ mod tests { .expect("replacement connection accepted") .unwrap(); assert_eq!(accepted.load(Ordering::Acquire), 2); - assert_eq!(requests.load(Ordering::Acquire), 4); + assert_eq!(requests.load(Ordering::Acquire), 5); // Another accept can report the same transport loss after recovery. // It must reuse the replacement without a third connection or replay. @@ -2279,7 +2516,7 @@ mod tests { .unwrap(); assert_eq!(client.connection_generation().await, recovered_generation); assert_eq!(accepted.load(Ordering::Acquire), 2); - assert_eq!(requests.load(Ordering::Acquire), 4); + assert_eq!(requests.load(Ordering::Acquire), 5); } #[test] @@ -2295,7 +2532,7 @@ mod tests { } #[tokio::test] - async fn mediation_transport_failure_recovers_without_deadlocking_the_cache() { + async fn mediation_stream_failure_retries_without_deadlocking_the_cache() { let certificate = test_certificate(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); @@ -2353,12 +2590,12 @@ mod tests { .await .expect("mediation recovery must not deadlock") .expect("mediation recovery must open a replacement session"); - tokio::time::timeout(Duration::from_secs(1), server) - .await - .expect("replacement physical connection must be accepted") - .unwrap(); + // The liveness probe keeps the healthy physical connection. + assert!(!server.is_finished(), "no replacement connection expected"); + server.abort(); assert_eq!(mediation_failures.load(Ordering::Acquire), 0); - assert_eq!(requests.load(Ordering::Acquire), 5); + // Attach, confirm, liveness confirm, then the successful mediation. + assert_eq!(requests.load(Ordering::Acquire), 4); } #[tokio::test] @@ -2413,8 +2650,10 @@ mod tests { request: vec![1, 2, 3], transport: openshell_isolation_interface::contract::DnsTransport::Udp, identity: crate::boundary_protocol::BinaryIdentityWire::Resolved { - binary_path: PathBuf::from("/usr/bin/dig"), - binary_digest: Some("a".repeat(64).parse().unwrap()), + executable: crate::boundary_protocol::ExecutableIdentityWire { + path: PathBuf::from("/usr/bin/dig"), + digest: Some("a".repeat(64).parse().unwrap()), + }, ancestors: Vec::new(), cmdline_paths: Vec::new(), }, @@ -2442,7 +2681,7 @@ mod tests { let query = session.accept_dns().await.unwrap(); assert_eq!(query.message, [1, 2, 3]); assert_eq!( - query.binary_identity.unwrap().binary_path, + query.binary_identity.unwrap().executable.path, PathBuf::from("/usr/bin/dig") ); query.response.send(Ok(vec![4, 5, 6])).unwrap(); diff --git a/crates/openshell-sandbox-backend/src/runtime/tests/flow_control.rs b/crates/openshell-sandbox-backend/src/runtime/tests/flow_control.rs new file mode 100644 index 0000000000..ad625aa91c --- /dev/null +++ b/crates/openshell-sandbox-backend/src/runtime/tests/flow_control.rs @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use tokio_stream::StreamExt as _; + +/// Stalled streams hold received data unread, as the boundary does when a +/// workload stops reading a relayed download. Other streams echo one chunk. +#[derive(Clone)] +struct StallingPeer { + stalled: Arc>>>, +} + +#[tonic::async_trait] +impl IsolationBoundary for StallingPeer { + type ExchangeStream = TestGrpcStream; + type MediateStream = TestGrpcStream; + + async fn exchange( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + let stall = request.metadata().get("x-stall").is_some(); + let mut inbound = request.into_inner(); + let (outbound, receiver) = tokio::sync::mpsc::channel(1); + if stall { + self.stalled.lock().unwrap().push(inbound); + } else { + tokio::spawn(async move { + if let Ok(Some(chunk)) = inbound.message().await { + let _ = outbound.send(Ok(chunk)).await; + } + }); + } + Ok(tonic::Response::new(Box::pin(ReceiverStream::new( + receiver, + )))) + } + + async fn mediate( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("flow-control fixture")) + } +} + +/// Mirror the boundary server's HTTP/2 settings. +async fn start_boundary(peer: StallingPeer) -> tonic::transport::Channel { + let certificate = test_certificate(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let config = certificate.server_config.clone(); + tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let tls = tokio_rustls::TlsAcceptor::from(config) + .accept(socket) + .await + .unwrap(); + let incoming = tokio_stream::iter([Ok::<_, std::io::Error>(TestTlsIo(Box::new(tls)))]) + .chain(tokio_stream::pending()); + tonic::transport::Server::builder() + .max_concurrent_streams(crate::boundary_protocol::BOUNDARY_MAX_CONCURRENT_STREAMS) + .initial_stream_window_size(crate::boundary_protocol::BOUNDARY_STREAM_WINDOW_BYTES) + .initial_connection_window_size( + crate::boundary_protocol::BOUNDARY_CONNECTION_WINDOW_BYTES, + ) + .add_service(IsolationBoundaryServer::new(peer)) + .serve_with_incoming(incoming) + .await + .unwrap(); + }); + let client = BoundaryClient::new( + tls_runtime_descriptor(address, certificate.client_tls), + test_bearer(&"a".repeat(32)), + ); + client.build_grpc_channel().await.unwrap().0 +} + +/// Push into one stalled stream until the peer stops granting credit. +async fn saturate(channel: tonic::transport::Channel) -> tokio::sync::mpsc::Sender { + let (sender, receiver) = tokio::sync::mpsc::channel(1); + let mut request = tonic::Request::new(ReceiverStream::new(receiver)); + request + .metadata_mut() + .insert("x-stall", "1".parse().unwrap()); + let mut client = IsolationBoundaryClient::new(channel); + tokio::spawn(async move { + let _response = client.exchange(request).await; + std::future::pending::<()>().await; + }); + let chunk = vec![0_u8; 32 * 1024]; + while tokio::time::timeout( + Duration::from_millis(300), + sender.send(BoundaryChunk { + data: chunk.clone(), + }), + ) + .await + .is_ok() + {} + sender +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stalled_relays_cannot_starve_other_streams() { + let channel = start_boundary(StallingPeer { + stalled: Arc::default(), + }) + .await; + let mut saturating = tokio::task::JoinSet::new(); + for _ in 0..MAX_NETWORK_STREAMS { + saturating.spawn(saturate(channel.clone())); + } + let mut stalled = Vec::new(); + while let Some(sender) = saturating.join_next().await { + stalled.push(sender.unwrap()); + } + + let auth: tonic::metadata::AsciiMetadataValue = "Bearer test".parse().unwrap(); + let exchange = async { + let mut stream = + open_grpc_client_stream_with_authorization(channel, GrpcStreamKind::Exchange, auth) + .await + .unwrap(); + stream.write_all(b"ping").await.unwrap(); + let mut pong = [0_u8; 4]; + stream.read_exact(&mut pong).await.unwrap(); + pong + }; + let pong = tokio::time::timeout(Duration::from_secs(3), exchange) + .await + .expect("stalled relay streams must not exhaust connection-level credit"); + assert_eq!(&pong, b"ping"); + drop(stalled); +} diff --git a/crates/openshell-sandbox-backend/src/runtime/tests/network_recovery.rs b/crates/openshell-sandbox-backend/src/runtime/tests/network_recovery.rs index 0ad9c7b441..aa4a1b1409 100644 --- a/crates/openshell-sandbox-backend/src/runtime/tests/network_recovery.rs +++ b/crates/openshell-sandbox-backend/src/runtime/tests/network_recovery.rs @@ -13,7 +13,18 @@ use tokio_stream::StreamExt as _; #[derive(Clone, Copy)] enum Reply { - Disconnect { pending_accepts: usize }, + Disconnect { + pending_accepts: usize, + }, + /// Like `Disconnect`, but the boundary retires the connection after the + /// transport closes, as the real disconnect callback can. + DisconnectLate, + /// Like `DisconnectLate`, but retirement waits for `PeerState::retire`. + DisconnectGated, + /// Close the first accept stream without a response on a live connection. + DropFirstAccept, + /// Stop answering on the first connection once it is confirmed. + Blackhole, Leaf(BoundaryErrorKind), Idle, } @@ -27,6 +38,8 @@ struct PeerState { events: Mutex>, accepting: tokio::sync::Notify, release: tokio::sync::Notify, + retire: tokio::sync::Notify, + rejected_attaches: AtomicUsize, } #[derive(Clone)] @@ -84,6 +97,12 @@ impl NetworkPeer { frame.extend_from_slice(&inbound.message().await?.expect("control request").data); } let envelope: RequestEnvelope = decode_frame(&frame).unwrap(); + if matches!(self.state.reply, Reply::Blackhole) + && self.connection == 1 + && *self.state.active.lock().unwrap() == Some(1) + { + return std::future::pending().await; + } let kind = match envelope.request { Request::Attach { .. } => "attach", Request::Confirm => "confirm", @@ -101,9 +120,10 @@ impl NetworkPeer { // This rejects reconnect storms that a stateless mock permits. let active = *self.state.active.lock().unwrap(); if active.is_some_and(|active| active != self.connection) { + self.state.rejected_attaches.fetch_add(1, Ordering::AcqRel); Response::Error { - kind: BoundaryErrorKind::Denied, - message: "credential epoch is already active".into(), + kind: BoundaryErrorKind::Unavailable, + message: "another connection with this epoch is still active".into(), } } else { self.attached.store(true, Ordering::Release); @@ -130,6 +150,15 @@ impl NetworkPeer { ); self.state.accepting.notify_one(); match self.state.reply { + Reply::DisconnectLate | Reply::DisconnectGated if self.connection == 1 => { + self.disconnect.notify_one(); + return std::future::pending().await; + } + Reply::DropFirstAccept + if self.state.first_accepts.fetch_add(1, Ordering::AcqRel) == 0 => + { + return Ok(()); + } Reply::Disconnect { pending_accepts } if self.connection == 1 => { if self.state.first_accepts.fetch_add(1, Ordering::AcqRel) + 1 == pending_accepts @@ -147,7 +176,11 @@ impl NetworkPeer { self.state.release.notified().await; network_response() } - Reply::Disconnect { .. } => network_response(), + Reply::Disconnect { .. } + | Reply::DisconnectLate + | Reply::DisconnectGated + | Reply::DropFirstAccept + | Reply::Blackhole => network_response(), } } _ => unreachable!(), @@ -197,8 +230,10 @@ impl NetworkPeer { fn network_response() -> Response { Response::NetworkConnected { identity: BinaryIdentityWire::Resolved { - binary_path: PathBuf::from("/usr/bin/curl"), - binary_digest: None, + executable: crate::boundary_protocol::ExecutableIdentityWire { + path: PathBuf::from("/usr/bin/curl"), + digest: None, + }, ancestors: Vec::new(), cmdline_paths: Vec::new(), }, @@ -236,6 +271,8 @@ impl Fixture { events: Mutex::new(Vec::new()), accepting: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), + retire: tokio::sync::Notify::new(), + rejected_attaches: AtomicUsize::new(0), }); let server_state = state.clone(); let server = tokio::spawn(async move { @@ -272,6 +309,17 @@ impl Fixture { _ = tokio::io::copy_bidirectional(&mut tls, &mut bridge) => {}, () = disconnect.notified() => {}, } + match state.reply { + Reply::DisconnectLate => { + drop(tls); + tokio::time::sleep(Duration::from_millis(300)).await; + } + Reply::DisconnectGated if connection == 1 => { + drop(tls); + state.retire.notified().await; + } + _ => {} + } let mut active = state.active.lock().unwrap(); if *active == Some(connection) { *active = None; @@ -315,7 +363,7 @@ impl Drop for Fixture { async fn verify_connection(mut pending: PendingTcpOpen) { assert_eq!(pending.destination, "203.0.113.1:443".parse().unwrap()); assert_eq!( - pending.binary_identity.unwrap().binary_path, + pending.binary_identity.unwrap().executable.path, PathBuf::from("/usr/bin/curl") ); assert_eq!( @@ -446,3 +494,113 @@ async fn tcp_accept_has_no_idle_operation_timeout() { .expect("idle acceptance must deliver the next connection"); assert_eq!(fixture.state.connections.load(Ordering::Acquire), 1); } + +#[tokio::test] +async fn tcp_accept_stream_failure_keeps_a_live_connection() { + let fixture = Fixture::new(Reply::DropFirstAccept).await; + tokio::time::timeout(Duration::from_secs(3), async { + verify_connection(fixture.source.accept_tcp().await.unwrap()).await; + }) + .await + .expect("a stream failure on a live connection must retry the accept"); + assert_eq!(fixture.state.connections.load(Ordering::Acquire), 1); + assert_eq!( + *fixture.state.events.lock().unwrap(), + vec![ + (1, "attach"), + (1, "confirm"), + (1, "accept"), + (1, "confirm"), + (1, "accept") + ] + ); +} + +#[tokio::test] +async fn tcp_accept_recovery_waits_for_the_boundary_to_retire_the_old_connection() { + let fixture = Fixture::new(Reply::DisconnectLate).await; + tokio::time::timeout(Duration::from_secs(5), async { + verify_connection(fixture.source.accept_tcp().await.unwrap()).await; + }) + .await + .unwrap_or_else(|error| { + panic!( + "reattach must retry until the old connection is retired: {error}; events: {:?}", + fixture.state.events.lock().unwrap() + ) + }); + let events = fixture.state.events.lock().unwrap().clone(); + assert!( + events + .iter() + .filter(|(connection, kind)| *connection > 1 && *kind == "attach") + .count() + > 1, + "expected a rejected same-epoch attach before recovery: {events:?}" + ); + assert_eq!(events.last(), Some(&(events.last().unwrap().0, "accept"))); +} + +#[tokio::test] +async fn recovery_closes_an_unresponsive_connection_before_reattaching() { + let fixture = Fixture::new(Reply::Blackhole).await; + let client = fixture.source.client.clone(); + let generation = client.connection_generation().await; + tokio::time::timeout( + CONNECTION_PROBE_TIMEOUT + Duration::from_secs(3), + client.recover_after_unavailable(generation), + ) + .await + .expect("recovery must not wait on a blackholed connection") + .expect("recovery must reattach after closing the old transport"); + assert_ne!(client.connection_generation().await, generation); + assert_eq!(*fixture.state.active.lock().unwrap(), Some(2)); + assert_eq!(fixture.state.connections.load(Ordering::Acquire), 2); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn requests_during_delayed_retirement_wait_for_the_confirmed_replacement() { + let fixture = Fixture::new(Reply::DisconnectGated).await; + let first = RemoteNetworkMediation { + client: fixture.source.client.clone(), + }; + let first = tokio::spawn(async move { first.accept_tcp().await }); + tokio::time::timeout(Duration::from_secs(3), async { + while fixture.state.rejected_attaches.load(Ordering::Acquire) == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("recovery must reach a same-epoch attach rejection"); + + // Start an ordinary request while recovery waits for retirement. + let second = RemoteNetworkMediation { + client: fixture.source.client.clone(), + }; + let second = tokio::spawn(async move { second.accept_tcp().await }); + tokio::time::sleep(Duration::from_millis(300)).await; + fixture.state.retire.notify_one(); + + for pending in [first, second] { + let pending = tokio::time::timeout(Duration::from_secs(5), pending) + .await + .expect("request must finish on the confirmed replacement") + .unwrap() + .unwrap(); + verify_connection(pending).await; + } + let events = fixture.state.events.lock().unwrap().clone(); + let confirmed: Vec = events + .iter() + .filter(|(connection, kind)| *connection > 1 && *kind == "confirm") + .map(|(connection, _)| *connection) + .collect(); + assert_eq!(confirmed.len(), 1, "one confirmed replacement: {events:?}"); + assert!( + events + .iter() + .filter(|(connection, kind)| *connection > 1 && *kind == "accept") + .all(|(connection, _)| *connection == confirmed[0]), + "no request may run on an unconfirmed connection: {events:?}" + ); +} diff --git a/crates/openshell-sandbox-backend/src/sandbox_auth.rs b/crates/openshell-sandbox-backend/src/sandbox_auth.rs index 8cd3878283..e9a72d5fd7 100644 --- a/crates/openshell-sandbox-backend/src/sandbox_auth.rs +++ b/crates/openshell-sandbox-backend/src/sandbox_auth.rs @@ -190,7 +190,10 @@ impl SandboxConnectionRegistry { if active.id == principal.connection_id && active.epoch == epoch { return Ok(None); } - if epoch <= active.epoch { + if epoch == active.epoch { + return Err(SandboxAuthError::ConnectionStillActive); + } + if epoch < active.epoch { return Err(SandboxAuthError::StaleCredentialEpoch); } } @@ -337,8 +340,11 @@ pub enum SandboxAuthError { WrongSandbox, #[error("authenticated runtime generation does not match this sandbox runtime")] WrongRuntimeGeneration, - #[error("Sandbox Protocol credential epoch is stale or already active")] + #[error("Sandbox Protocol credential epoch is stale")] StaleCredentialEpoch, + /// Retryable: the previous connection's disconnect has not been observed yet. + #[error("another Sandbox Protocol connection with this credential epoch is still active")] + ConnectionStillActive, #[error("sandbox runtime is already bound to another supervisor process")] WrongSupervisorInstance, #[error("Sandbox Protocol connection has not completed attach")] @@ -467,7 +473,19 @@ mod tests { assert_eq!(registry.confirm(&first), Ok(None)); assert_eq!(registry.confirm(&first), Ok(None)); + let second_id = SandboxConnectionId::new(); + let second = first_authenticator + .authenticate(second_id, &metadata(first_token.token.expose_secret())) + .expect("second principal"); + assert_eq!( + registry.attach(&second, instance), + Err(SandboxAuthError::ConnectionStillActive) + ); + assert!(registry.disconnect(first_id)); + assert_eq!(registry.attach(&second, instance), Ok(None)); + assert_eq!(registry.confirm(&second), Ok(None)); + assert!(registry.disconnect(second_id)); assert_eq!(registry.attach(&first, instance), Ok(None)); assert_eq!(registry.confirm(&first), Ok(None)); registry.mark_terminal(); diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 9d1f477241..3d85d730a7 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -49,8 +49,8 @@ mod linux { isolation_boundary_server::{IsolationBoundary, IsolationBoundaryServer}, }; use openshell_sandbox_backend::sandbox_auth::{ - SandboxConnectionId, SandboxConnectionRegistry, SandboxProtocolAuthenticator, - SandboxProtocolPrincipal, + SandboxAuthError, SandboxConnectionId, SandboxConnectionRegistry, + SandboxProtocolAuthenticator, SandboxProtocolPrincipal, }; use openshell_sandbox_backend::{ ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM, GPU_RESOURCE_CLAIM, @@ -59,7 +59,8 @@ mod linux { use tokio_stream::wrappers::ReceiverStream; use openshell_sandbox_backend::boundary_protocol::{ - AgentSpecWire, BinaryIdentityWire, BoundaryConfig, BoundaryErrorKind, + AgentSpecWire, BOUNDARY_CONNECTION_WINDOW_BYTES, BOUNDARY_MAX_CONCURRENT_STREAMS, + BOUNDARY_STREAM_WINDOW_BYTES, BinaryIdentityWire, BoundaryConfig, BoundaryErrorKind, BoundaryListener as BoundaryListenerConfig, DnsQueryResultWire, ExecSpecWire, ExitStatusWire, MediationTimingWire, NativeLinuxSandboxAuditEvidence, OutputWindowWire, ProcessKindWire, ProcessSnapshotWire, Request, RequestEnvelope, Response, ResponseEnvelope, @@ -598,12 +599,9 @@ mod linux { let result = tonic::transport::Server::builder() .http2_keepalive_interval(Some(CONTROL_KEEPALIVE_INTERVAL)) .http2_keepalive_timeout(Some(CONTROL_KEEPALIVE_TIMEOUT)) - .max_concurrent_streams( - u32::try_from(MAX_CONTROL_CONNECTIONS) - .map_err(|error| format!("invalid control connection limit: {error}"))?, - ) - .initial_stream_window_size(16 * 1024 * 1024) - .initial_connection_window_size(16 * 1024 * 1024) + .max_concurrent_streams(BOUNDARY_MAX_CONCURRENT_STREAMS) + .initial_stream_window_size(BOUNDARY_STREAM_WINDOW_BYTES) + .initial_connection_window_size(BOUNDARY_CONNECTION_WINDOW_BYTES) .add_service( IsolationBoundaryServer::new(GrpcBoundaryService { runtime: runtime.clone(), @@ -1277,17 +1275,24 @@ mod linux { }; let is_attach = supervisor_instance_id.is_some(); let is_confirm = matches!(&request.request, Request::Confirm); - let response = ResponseEnvelope { + let mut response = ResponseEnvelope { request_id: request.request_id.clone(), response: runtime.dispatch(request), }; + // Report commit failures to the supervisor: a silently closed stream + // is indistinguishable from transport loss. if is_attach && matches!(&response.response, Response::Attached { .. }) { let supervisor_instance_id = supervisor_instance_id .ok_or_else(|| "attach request lost supervisor instance identity".to_string())?; - runtime.commit_attach(principal, supervisor_instance_id)?; + if let Err((kind, message)) = runtime.commit_attach(principal, supervisor_instance_id) { + response.response = guest_error(kind, message); + } } - if is_confirm && matches!(&response.response, Response::Confirmed { .. }) { - runtime.commit_confirm(principal)?; + if is_confirm + && matches!(&response.response, Response::Confirmed { .. }) + && let Err((kind, message)) = runtime.commit_confirm(principal) + { + response.response = guest_error(kind, message); } write_frame(&mut stream, &response) .map_err(|error| format!("write control frame: {error}"))?; @@ -1637,22 +1642,22 @@ mod linux { &self, principal: &SandboxProtocolPrincipal, supervisor_instance_id: openshell_sandbox_backend::boundary_protocol::SupervisorInstanceId, - ) -> Result<(), String> { + ) -> Result<(), CommitError> { if let Some(replaced) = self .connections .attach(principal, supervisor_instance_id) - .map_err(|error| error.to_string())? + .map_err(connection_auth_error)? { self.close_connection(replaced); } Ok(()) } - fn commit_confirm(&self, principal: &SandboxProtocolPrincipal) -> Result<(), String> { + fn commit_confirm(&self, principal: &SandboxProtocolPrincipal) -> Result<(), CommitError> { let replaced = self .connections .confirm(principal) - .map_err(|error| error.to_string())?; + .map_err(connection_auth_error)?; let process = { let state = lock(&self.state); match &*state { @@ -1669,13 +1674,19 @@ mod linux { SupervisorConnectionState::Terminating | SupervisorConnectionState::Terminal ) { self.connections.mark_terminal(); - return Err("sandbox session is terminating".to_string()); + return Err(( + BoundaryErrorKind::Terminated, + "sandbox session is terminating".to_string(), + )); } if matches!(*connection, SupervisorConnectionState::Frozen { .. }) && let Some(process) = process { if !process.boundary_runtime.resume() { - return Err("frozen workload could not be resumed".to_string()); + return Err(( + BoundaryErrorKind::Process, + "frozen workload could not be resumed".to_string(), + )); } tracing::info!( connection_id = ?principal.connection_id(), @@ -3066,6 +3077,17 @@ mod linux { ) } + type CommitError = (BoundaryErrorKind, String); + + fn connection_auth_error(error: SandboxAuthError) -> CommitError { + let kind = match error { + SandboxAuthError::ConnectionStillActive => BoundaryErrorKind::Unavailable, + SandboxAuthError::TerminalSession => BoundaryErrorKind::Terminated, + _ => BoundaryErrorKind::Denied, + }; + (kind, error.to_string()) + } + fn guest_error(kind: BoundaryErrorKind, message: impl Into) -> Response { Response::Error { kind, @@ -4032,6 +4054,19 @@ mod linux { *lock(&runtime.supervisor_connection), SupervisorConnectionState::Connected(first_id) ); + let early = runtime + .authenticate_request( + SandboxConnectionId::new(), + bearer_request((), &token).metadata(), + ) + .expect("early replacement principal"); + assert!( + matches!( + runtime.commit_attach(&early, supervisor_instance_id), + Err((BoundaryErrorKind::Unavailable, _)) + ), + "reattach before the old transport is retired must be retryable" + ); runtime.transport_disconnected(first_id); let connection_state = *lock(&runtime.supervisor_connection); @@ -4054,7 +4089,7 @@ mod linux { .expect("reattach replacement"); assert_eq!( runtime.connections.require_active(&replacement), - Err(openshell_sandbox_backend::sandbox_auth::SandboxAuthError::ConnectionNotAttached) + Err(SandboxAuthError::ConnectionNotAttached) ); runtime .commit_confirm(&replacement) @@ -4130,7 +4165,7 @@ mod linux { ); assert_eq!( runtime.connections.require_active(&principal), - Err(openshell_sandbox_backend::sandbox_auth::SandboxAuthError::TerminalSession) + Err(SandboxAuthError::TerminalSession) ); } diff --git a/crates/openshell-server/src/auth/authenticator.rs b/crates/openshell-server/src/auth/authenticator.rs index 5511ef79db..fb691d2e4b 100644 --- a/crates/openshell-server/src/auth/authenticator.rs +++ b/crates/openshell-server/src/auth/authenticator.rs @@ -13,7 +13,8 @@ //! the chain — fail-closed. //! //! Live authenticators slotting into the chain: -//! - [`super::sandbox_jwt::SandboxJwtAuthenticator`] — gateway-minted JWTs +//! - [`super::sandbox_jwt::SandboxSessionJwtAuthenticator`] — generation-bound +//! gateway session JWTs //! - [`super::compute_driver::ComputeDriverAuthenticator`] — driver-native //! sandbox bootstrap credentials (path-scoped to `IssueSandboxToken`) //! - [`super::oidc::OidcAuthenticator`] — user OIDC Bearer tokens diff --git a/crates/openshell-server/src/auth/http.rs b/crates/openshell-server/src/auth/http.rs index df95f35fd7..8f0bb7d11c 100644 --- a/crates/openshell-server/src/auth/http.rs +++ b/crates/openshell-server/src/auth/http.rs @@ -84,7 +84,7 @@ async fn gateway_discovery_handler( uri: Uri, ) -> impl IntoResponse { gateway_discovery_response( - state.sandbox_jwt_authenticator.as_deref(), + state.extension_jwt_issuer.as_deref(), &document_base_url(&headers, &uri), ) } @@ -113,10 +113,10 @@ fn document_base_url(headers: &HeaderMap, uri: &Uri) -> Option { } fn gateway_discovery_response( - authenticator: Option<&crate::auth::sandbox_jwt::SandboxJwtAuthenticator>, + issuer: Option<&crate::auth::sandbox_jwt::ExtensionJwtIssuer>, base_url: &Option, ) -> axum::response::Response { - let Some(authenticator) = authenticator else { + let Some(issuer) = issuer else { return StatusCode::NOT_FOUND.into_response(); }; let Some(base_url) = base_url else { @@ -126,7 +126,7 @@ fn gateway_discovery_response( return StatusCode::BAD_REQUEST.into_response(); }; Json(serde_json::json!({ - "issuer": authenticator.issuer(), + "issuer": issuer.issuer(), "jwks_uri": format!("{base_url}/.well-known/jwks.json"), "id_token_signing_alg_values_supported": ["EdDSA"], "response_types_supported": ["id_token"], @@ -145,15 +145,15 @@ fn gateway_discovery_response( /// gateway from which an integration bootstraps this document; integrations /// then cache keys by `kid` and refresh when an unfamiliar `kid` appears. async fn gateway_jwks_handler(State(state): State>) -> impl IntoResponse { - gateway_jwks_response(state.sandbox_jwt_authenticator.as_deref()) + gateway_jwks_response(state.extension_jwt_issuer.as_deref()) } fn gateway_jwks_response( - authenticator: Option<&crate::auth::sandbox_jwt::SandboxJwtAuthenticator>, + issuer: Option<&crate::auth::sandbox_jwt::ExtensionJwtIssuer>, ) -> axum::response::Response { - authenticator.map_or_else( + issuer.map_or_else( || StatusCode::NOT_FOUND.into_response(), - |authenticator| Json(authenticator.jwks()).into_response(), + |issuer| Json(issuer.jwks()).into_response(), ) } @@ -571,6 +571,7 @@ mod tests { use super::*; use axum::body::to_bytes; use openshell_bootstrap::jwt::generate_jwt_key; + use std::time::Duration; #[test] fn extract_cookie_finds_value() { @@ -687,16 +688,18 @@ mod tests { #[test] fn jwks_response_publishes_configured_gateway_key() { let material = generate_jwt_key().expect("key"); - let authenticator = crate::auth::sandbox_jwt::SandboxJwtAuthenticator::from_pem( + let issuer = crate::auth::sandbox_jwt::ExtensionJwtIssuer::from_pem( + material.signing_key_pem.as_bytes(), material.public_key_pem.as_bytes(), material.kid.clone(), "gateway-a", + Duration::from_mins(15), ) - .expect("authenticator"); + .expect("issuer"); - let response = gateway_jwks_response(Some(&authenticator)); + let response = gateway_jwks_response(Some(&issuer)); assert_eq!(response.status(), StatusCode::OK); - let key = &authenticator.jwks().keys[0]; + let key = &issuer.jwks().keys[0]; assert_eq!(key.kid, material.kid); assert_eq!(key.kty, "OKP"); assert_eq!(key.crv, "Ed25519"); @@ -709,21 +712,23 @@ mod tests { assert_eq!(gateway_jwks_response(None).status(), StatusCode::NOT_FOUND); } - fn test_authenticator() -> crate::auth::sandbox_jwt::SandboxJwtAuthenticator { + fn test_issuer() -> crate::auth::sandbox_jwt::ExtensionJwtIssuer { let material = generate_jwt_key().expect("key"); - crate::auth::sandbox_jwt::SandboxJwtAuthenticator::from_pem( + crate::auth::sandbox_jwt::ExtensionJwtIssuer::from_pem( + material.signing_key_pem.as_bytes(), material.public_key_pem.as_bytes(), material.kid, "gateway-a", + Duration::from_mins(15), ) - .expect("authenticator") + .expect("issuer") } #[tokio::test] async fn discovery_document_advertises_issuer_identity_and_absolute_jwks_uri() { - let authenticator = test_authenticator(); + let issuer = test_issuer(); let response = gateway_discovery_response( - Some(&authenticator), + Some(&issuer), &Some("https://gateway.example:8443".to_string()), ); assert_eq!(response.status(), StatusCode::OK); @@ -755,7 +760,7 @@ mod tests { // A relative jwks_uri could be resolved against the wrong origin, so // refuse to emit a document rather than emit an ambiguous one. assert_eq!( - gateway_discovery_response(Some(&test_authenticator()), &None).status(), + gateway_discovery_response(Some(&test_issuer()), &None).status(), StatusCode::BAD_REQUEST ); } diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs index 41520f3bb8..390e4861eb 100644 --- a/crates/openshell-server/src/auth/principal.rs +++ b/crates/openshell-server/src/auth/principal.rs @@ -75,8 +75,8 @@ pub struct PeerPrincipal { #[derive(Debug, Clone)] #[allow(dead_code)] pub enum SandboxIdentitySource { - /// Gateway-minted JWT validated against the gateway's signing key. - /// Produced by [`super::sandbox_jwt::SandboxJwtAuthenticator`]. + /// Generation-bound gateway JWT validated against the persisted runtime + /// identity by [`super::sandbox_jwt::SandboxSessionJwtAuthenticator`]. BootstrapJwt { issuer: String }, /// Per-sandbox client certificate. Reserved for channel-bound sandbox /// identity. diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 8593c01b33..d5f1cafa11 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -1,16 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Gateway-minted per-sandbox JWTs. +//! Gateway-minted session and extension JWTs. //! -//! The gateway signs an Ed25519 JWT for each sandbox at create time and -//! the sandbox supervisor presents it as `Authorization: Bearer ` on -//! supervisor-to-gateway gRPC calls. This module implements both sides of the -//! gateway-controlled token: -//! - [`SandboxJwtIssuer`] mints fresh tokens (called from -//! `handle_create_sandbox` and the `IssueSandboxToken` RPC). -//! - [`SandboxJwtAuthenticator`] validates tokens on inbound requests and -//! produces a [`Principal::Sandbox`] with [`SandboxIdentitySource::BootstrapJwt`]. +//! [`SandboxSessionJwtAuthority`] mints and verifies generation-bound sandbox +//! session tokens. [`ExtensionJwtIssuer`] mints explicitly typed credentials +//! for supervisor middleware and gateway interceptors and publishes the +//! corresponding JWKS metadata. //! //! Algorithm: `EdDSA` (Ed25519). Pinned via `Validation::algorithms` to //! prevent algorithm-confusion attacks. @@ -19,21 +15,19 @@ use super::authenticator::Authenticator; use super::principal::{Principal, SandboxIdentitySource, SandboxPrincipal}; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use jsonwebtoken::{ - Algorithm, DecodingKey, EncodingKey, Header, Validation, decode, decode_header, encode, -}; +use jsonwebtoken::{Algorithm, EncodingKey, Header, decode_header, encode}; pub use openshell_extension_core::{ EXTENSION_JWT_TYP, ExtensionAudience, ExtensionCallerKind, ExtensionJwtClaims, MAX_EXTENSION_TOKEN_TTL, }; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use std::{ io::Cursor, sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; use tonic::Status; -use tracing::{debug, warn}; +use tracing::warn; use x509_parser::{oid_registry::OID_SIG_ED25519, prelude::FromDer, x509::SubjectPublicKeyInfo}; use openshell_core::SandboxSessionId; @@ -49,7 +43,6 @@ use openshell_core::sandbox_generation::SandboxGenerationId; /// reuse the same subject namespace without breaking handler equality /// checks. const SPIFFE_SUBJECT_PREFIX: &str = "spiffe://openshell/sandbox/"; -const SANDBOX_JWT_EXP_LEEWAY_SECS: i64 = 60; /// Public JSON Web Key Set served by the gateway. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] @@ -69,39 +62,22 @@ pub struct GatewayJwk { pub x: String, } -/// JWT claim set serialized in every gateway-minted sandbox token. -#[derive(Debug, Serialize, Deserialize)] -pub struct SandboxJwtClaims { - /// `spiffe://openshell/sandbox/`. SPIFFE-shaped for forward - /// compatibility with channel-bound identity (per-sandbox cert / SPIRE). - pub sub: String, - /// Gateway identity (`openshell-gateway:`). Both `iss` and - /// `aud` use the same value so any future replicas of the same - /// deployment validate each others' tokens without configuration. - pub iss: String, - pub aud: String, - pub iat: i64, - pub exp: i64, - /// Canonical sandbox UUID, denormalized from `sub` for cheap parsing - /// without a SPIFFE library. - pub sandbox_id: String, -} - -/// Mints fresh sandbox JWTs. -pub struct SandboxJwtIssuer { +/// Mints typed extension JWTs and publishes their verification metadata. +pub struct ExtensionJwtIssuer { encoding_key: EncodingKey, kid: String, issuer: String, - audience: String, - ttl: Option, + gateway_audience: String, + ttl: Duration, + jwks: GatewayJwks, } -impl std::fmt::Debug for SandboxJwtIssuer { +impl std::fmt::Debug for ExtensionJwtIssuer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SandboxJwtIssuer") + f.debug_struct("ExtensionJwtIssuer") .field("kid", &self.kid) .field("issuer", &self.issuer) - .field("audience", &self.audience) + .field("gateway_audience", &self.gateway_audience) .field("ttl", &self.ttl) .finish_non_exhaustive() } @@ -330,57 +306,31 @@ impl Authenticator for SandboxSessionJwtAuthenticator { } } -impl SandboxJwtIssuer { +impl ExtensionJwtIssuer { pub fn from_pem( signing_key_pem: &[u8], + public_key_pem: &[u8], kid: String, gateway_id: &str, - ttl: Option, + ttl: Duration, ) -> Result { crate::install_jsonwebtoken_crypto_provider(); - if ttl.is_some_and(|ttl| ttl.is_zero()) { - return Err("sandbox token TTL must be positive when configured".to_string()); + if ttl.is_zero() { + return Err("gateway token TTL must be positive".to_string()); } let encoding_key = EncodingKey::from_ed_pem(signing_key_pem) .map_err(|e| format!("failed to parse Ed25519 signing key PEM: {e}"))?; + let jwks = GatewayJwks::from_public_key_pem(public_key_pem, kid.clone())?; let identity = format!("openshell-gateway:{gateway_id}"); Ok(Self { encoding_key, kid, issuer: identity.clone(), - audience: identity, + gateway_audience: identity, ttl, - }) - } - - /// Mint a fresh token for `sandbox_id`. - #[allow(clippy::result_large_err)] // `tonic::Status` is the natural error here - pub fn mint(&self, sandbox_id: &str) -> Result { - crate::install_jsonwebtoken_crypto_provider(); - - let now = now_secs(); - let exp = self.ttl.map_or(0, |ttl| { - now.saturating_add(i64::try_from(ttl.as_secs()).unwrap_or(3_600)) - }); - let claims = SandboxJwtClaims { - sub: format!("{SPIFFE_SUBJECT_PREFIX}{sandbox_id}"), - iss: self.issuer.clone(), - aud: self.audience.clone(), - iat: now, - exp, - sandbox_id: sandbox_id.to_string(), - }; - let mut header = Header::new(Algorithm::EdDSA); - header.kid = Some(self.kid.clone()); - let token = encode(&header, &claims, &self.encoding_key).map_err(|e| { - warn!(error = %e, "failed to mint sandbox JWT"); - Status::internal("failed to mint sandbox token") - })?; - Ok(MintedToken { - token, - expires_at_ms: exp.saturating_mul(1000), + jwks, }) } @@ -417,7 +367,7 @@ impl SandboxJwtIssuer { issued_at: i64, token_id: uuid::Uuid, ) -> Result { - if audience.as_str() == self.audience { + if audience.as_str() == self.gateway_audience { return Err(Status::invalid_argument( "extension audience must not equal the gateway sandbox audience", )); @@ -472,49 +422,12 @@ impl SandboxJwtIssuer { }) } - pub fn sandbox_token_ttl(&self) -> Option { + pub const fn token_ttl(&self) -> Duration { self.ttl } -} - -/// Authenticator that validates gateway-minted sandbox JWTs. -pub struct SandboxJwtAuthenticator { - decoding_key: DecodingKey, - kid: String, - issuer: String, - audience: String, - jwks: GatewayJwks, -} - -impl std::fmt::Debug for SandboxJwtAuthenticator { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SandboxJwtAuthenticator") - .field("kid", &self.kid) - .field("issuer", &self.issuer) - .field("audience", &self.audience) - .finish_non_exhaustive() - } -} - -impl SandboxJwtAuthenticator { - pub fn from_pem(public_key_pem: &[u8], kid: String, gateway_id: &str) -> Result { - crate::install_jsonwebtoken_crypto_provider(); - - let decoding_key = DecodingKey::from_ed_pem(public_key_pem) - .map_err(|e| format!("failed to parse Ed25519 public key PEM: {e}"))?; - let jwks = GatewayJwks::from_public_key_pem(public_key_pem, kid.clone())?; - let identity = format!("openshell-gateway:{gateway_id}"); - Ok(Self { - decoding_key, - kid, - issuer: identity.clone(), - audience: identity, - jwks, - }) - } /// Return the public signing keys integrations use to verify extension - /// tokens. No private key material is retained by this type. + /// tokens. #[must_use] pub const fn jwks(&self) -> &GatewayJwks { &self.jwks @@ -525,56 +438,6 @@ impl SandboxJwtAuthenticator { pub fn issuer(&self) -> &str { &self.issuer } - - #[allow(clippy::result_large_err)] - fn validate_bearer(&self, token: &str) -> Result, Status> { - crate::install_jsonwebtoken_crypto_provider(); - - let header = decode_header(token).map_err(|e| { - debug!(error = %e, "sandbox JWT header decode failed"); - Status::unauthenticated("invalid token") - })?; - - // Extension credentials share this signing key during alpha, but are - // never valid sandbox admission credentials. Check the explicit type - // before `kid` fallthrough so another authenticator cannot accept one. - // Legacy and untyped sandbox JWTs remain valid for rolling upgrades. - if header.typ.as_deref() == Some(EXTENSION_JWT_TYP) { - return Err(Status::unauthenticated( - "extension tokens cannot authenticate to the gateway", - )); - } - - // Fall through to other authenticators when the kid does not match — - // OIDC issuers may share the Bearer slot. - if header.kid.as_deref() != Some(self.kid.as_str()) { - return Ok(None); - } - if !matches!(header.alg, Algorithm::EdDSA) { - return Ok(None); - } - - let mut validation = Validation::new(Algorithm::EdDSA); - validation.algorithms = vec![Algorithm::EdDSA]; - validation.set_issuer(&[&self.issuer]); - validation.set_audience(&[&self.audience]); - validation.set_required_spec_claims(&["iss", "aud", "exp", "sub"]); - validation.validate_exp = false; - - let data = - decode::(token, &self.decoding_key, &validation).map_err(|e| { - debug!(error = %e, "sandbox JWT validation failed"); - Status::unauthenticated(format!("invalid token: {e}")) - })?; - - let claims = data.claims; - validate_exp(claims.exp)?; - Ok(Some(Principal::Sandbox(SandboxPrincipal { - sandbox_id: claims.sandbox_id, - source: SandboxIdentitySource::BootstrapJwt { issuer: claims.iss }, - trust_domain: Some("openshell".to_string()), - }))) - } } impl GatewayJwks { @@ -607,38 +470,6 @@ impl GatewayJwks { } } -#[async_trait] -impl Authenticator for SandboxJwtAuthenticator { - async fn authenticate( - &self, - headers: &http::HeaderMap, - _path: &str, - ) -> Result, Status> { - let Some(token) = headers - .get("authorization") - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.strip_prefix("Bearer ")) - else { - return Ok(None); - }; - self.validate_bearer(token) - } -} - -#[allow(clippy::result_large_err)] -fn validate_exp(exp: i64) -> Result<(), Status> { - if exp == 0 { - return Ok(()); - } - - if exp < now_secs().saturating_sub(SANDBOX_JWT_EXP_LEEWAY_SECS) { - debug!("sandbox JWT expired"); - return Err(Status::unauthenticated("invalid token: ExpiredSignature")); - } - - Ok(()) -} - fn now_secs() -> i64 { i64::try_from( SystemTime::now() @@ -651,229 +482,48 @@ fn now_secs() -> i64 { #[cfg(test)] mod tests { use super::*; + use jsonwebtoken::{DecodingKey, Validation, decode}; use openshell_bootstrap::jwt::generate_jwt_key; - fn header_map_with_bearer(token: &str) -> http::HeaderMap { - let mut h = http::HeaderMap::new(); - h.insert( - "authorization", - http::HeaderValue::from_str(&format!("Bearer {token}")).unwrap(), - ); - h - } - - fn pair() -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { - pair_with_ttl(Some(Duration::from_hours(1))) - } - - fn pair_with_ttl(ttl: Option) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { + fn issuer(ttl: Duration) -> ExtensionJwtIssuer { let mat = generate_jwt_key().expect("jwt key"); - let issuer = SandboxJwtIssuer::from_pem( + ExtensionJwtIssuer::from_pem( mat.signing_key_pem.as_bytes(), - mat.kid.clone(), - "test-gateway", - ttl, - ) - .unwrap(); - let auth = SandboxJwtAuthenticator::from_pem( mat.public_key_pem.as_bytes(), mat.kid, "test-gateway", + ttl, ) - .unwrap(); - (issuer, auth) + .expect("issuer") } fn extension_audience(value: &str) -> ExtensionAudience { ExtensionAudience::new(value).expect("valid extension audience") } - #[tokio::test] - async fn mint_and_validate_round_trip() { - let (issuer, auth) = pair(); - let minted = issuer.mint("sandbox-a").unwrap(); - assert!(minted.expires_at_ms > 0); - let principal = auth - .authenticate(&header_map_with_bearer(&minted.token), "/anything") - .await - .unwrap() - .expect("expected principal"); - match principal { - Principal::Sandbox(p) => { - assert_eq!(p.sandbox_id, "sandbox-a"); - match p.source { - SandboxIdentitySource::BootstrapJwt { issuer: iss } => { - assert_eq!(iss, "openshell-gateway:test-gateway"); - } - other => panic!("unexpected source: {other:?}"), - } - } - _ => panic!("expected Sandbox principal"), - } - } - - #[tokio::test] - async fn extension_token_cannot_authenticate_as_a_sandbox() { - let mat = generate_jwt_key().expect("jwt key"); - let issuer = SandboxJwtIssuer::from_pem( - mat.signing_key_pem.as_bytes(), - mat.kid.clone(), - "test-gateway", - Some(Duration::from_hours(1)), - ) - .expect("issuer"); - let auth = SandboxJwtAuthenticator::from_pem( - mat.public_key_pem.as_bytes(), - mat.kid.clone(), - "test-gateway", - ) - .expect("authenticator"); - - // Build the token directly to model a credential minted before the - // reserved-audience guard was added. Its claims otherwise satisfy the - // sandbox authenticator's issuer, audience, subject, and expiry checks. - let sandbox_id = "sandbox-a"; - let now = now_secs(); - let claims = ExtensionJwtClaims { - iss: "openshell-gateway:test-gateway".to_string(), - aud: "openshell-gateway:test-gateway".to_string(), - sub: format!("{SPIFFE_SUBJECT_PREFIX}{sandbox_id}"), - iat: now, - exp: now + 300, - jti: uuid::Uuid::new_v4().to_string(), - caller_kind: ExtensionCallerKind::Supervisor, - sandbox_id: Some(sandbox_id.to_string()), - }; - let mut header = Header::new(Algorithm::EdDSA); - header.kid = Some(mat.kid); - header.typ = Some(EXTENSION_JWT_TYP.to_string()); - let token = encode(&header, &claims, &issuer.encoding_key).expect("extension token"); - - let error = auth - .authenticate(&header_map_with_bearer(&token), "/anything") - .await - .expect_err("extension token must not authenticate as a sandbox"); - assert_eq!(error.code(), tonic::Code::Unauthenticated); - assert_eq!( - error.message(), - "extension tokens cannot authenticate to the gateway" - ); - } - - #[tokio::test] - async fn ttl_none_mints_non_expiring_token() { - let (issuer, auth) = pair_with_ttl(None); - let minted = issuer.mint("sandbox-never").unwrap(); - assert_eq!(minted.expires_at_ms, 0); - - let principal = auth - .authenticate(&header_map_with_bearer(&minted.token), "/anything") - .await - .unwrap() - .expect("exp=0 token should authenticate"); - assert!(matches!(principal, Principal::Sandbox(_))); - - let mut validation = Validation::new(Algorithm::EdDSA); - validation.algorithms = vec![Algorithm::EdDSA]; - validation.set_issuer(&["openshell-gateway:test-gateway"]); - validation.set_audience(&["openshell-gateway:test-gateway"]); - validation.set_required_spec_claims(&["iss", "aud", "exp", "sub"]); - validation.validate_exp = false; - let decoded = decode::(&minted.token, &auth.decoding_key, &validation) - .expect("token should decode"); - assert_eq!(decoded.claims.exp, 0); - } - #[test] - fn ttl_some_zero_is_rejected() { + fn zero_ttl_is_rejected() { let mat = generate_jwt_key().expect("jwt key"); - let error = SandboxJwtIssuer::from_pem( + let error = ExtensionJwtIssuer::from_pem( mat.signing_key_pem.as_bytes(), + mat.public_key_pem.as_bytes(), mat.kid, "test-gateway", - Some(Duration::ZERO), + Duration::ZERO, ) - .expect_err("Some(Duration::ZERO) must not reintroduce a sentinel"); + .expect_err("Duration::ZERO must not reintroduce a sentinel"); assert!(error.contains("must be positive")); } - #[tokio::test] - async fn token_signed_by_other_key_is_rejected() { - let (_, auth_a) = pair(); - let (issuer_b, _) = pair(); // different keypair - let minted = issuer_b.mint("sandbox-b").unwrap(); - // The token has a different `kid` than auth_a expects, so the - // authenticator yields None (lets the chain fall through). That is - // the documented behavior for cross-issuer Bearer headers. - let result = auth_a - .authenticate(&header_map_with_bearer(&minted.token), "/anything") - .await - .unwrap(); - assert!(result.is_none(), "different kid must fall through"); - } - - #[tokio::test] - async fn missing_bearer_yields_none() { - let (_, auth) = pair(); - let result = auth - .authenticate(&http::HeaderMap::new(), "/anything") - .await - .unwrap(); - assert!(result.is_none()); - } - - #[tokio::test] - async fn malformed_token_is_rejected() { - let (_, auth) = pair(); - let err = auth - .authenticate(&header_map_with_bearer("not.a.jwt"), "/anything") - .await - .expect_err("malformed must reject"); - assert_eq!(err.code(), tonic::Code::Unauthenticated); - } - - #[tokio::test] - async fn expired_token_is_rejected() { - // Mint a token whose iat is far in the past so its TTL window is - // already closed by `now`. We sign the JWT directly with the same - // signing key to bypass the issuer's TTL-vs-now coupling. - let mat = generate_jwt_key().unwrap(); - let issuer = SandboxJwtIssuer::from_pem( - mat.signing_key_pem.as_bytes(), - mat.kid.clone(), - "g", - Some(Duration::from_hours(1)), - ) - .unwrap(); - let auth = - SandboxJwtAuthenticator::from_pem(mat.public_key_pem.as_bytes(), mat.kid.clone(), "g") - .unwrap(); - let claims = SandboxJwtClaims { - sub: format!("{SPIFFE_SUBJECT_PREFIX}sandbox-c"), - iss: "openshell-gateway:g".to_string(), - aud: "openshell-gateway:g".to_string(), - iat: now_secs() - 7200, - exp: now_secs() - 3600, - sandbox_id: "sandbox-c".to_string(), - }; - let mut header = Header::new(Algorithm::EdDSA); - header.kid = Some(mat.kid); - let token = encode(&header, &claims, &issuer.encoding_key).unwrap(); - let err = auth - .authenticate(&header_map_with_bearer(&token), "/anything") - .await - .expect_err("expired token must reject"); - assert_eq!(err.code(), tonic::Code::Unauthenticated); - } - #[test] fn extension_tokens_have_exact_audience_and_caller_identity() { let mat = generate_jwt_key().expect("jwt key"); - let issuer = SandboxJwtIssuer::from_pem( + let issuer = ExtensionJwtIssuer::from_pem( mat.signing_key_pem.as_bytes(), + mat.public_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - None, + Duration::from_mins(15), ) .expect("issuer"); let decoding_key = DecodingKey::from_ed_pem(mat.public_key_pem.as_bytes()).unwrap(); @@ -916,16 +566,8 @@ mod tests { } #[test] - fn extension_tokens_are_explicitly_typed_and_sandbox_tokens_are_not() { - let mat = generate_jwt_key().expect("jwt key"); - let issuer = SandboxJwtIssuer::from_pem( - mat.signing_key_pem.as_bytes(), - mat.kid.clone(), - "gateway-a", - Some(Duration::from_hours(1)), - ) - .expect("issuer"); - + fn extension_tokens_are_explicitly_typed() { + let issuer = issuer(Duration::from_hours(1)); let extension = issuer .mint_extension_token( &extension_audience("urn:openshell:extension:middleware:scanner"), @@ -938,23 +580,19 @@ mod tests { decode_header(&extension.token).unwrap().typ.as_deref(), Some(EXTENSION_JWT_TYP) ); - - // The discriminator is only useful if the sandbox bootstrap token does - // not carry it. A verifier requiring `openshell-ext+jwt` must reject a - // gateway admission credential even though both share a signing key. - let sandbox = issuer.mint("sandbox-a").expect("sandbox token"); - assert_ne!( - decode_header(&sandbox.token).unwrap().typ.as_deref(), - Some(EXTENSION_JWT_TYP) - ); } #[test] fn extension_token_rejects_wrong_audience() { let mat = generate_jwt_key().expect("jwt key"); - let issuer = - SandboxJwtIssuer::from_pem(mat.signing_key_pem.as_bytes(), mat.kid, "gateway-a", None) - .expect("issuer"); + let issuer = ExtensionJwtIssuer::from_pem( + mat.signing_key_pem.as_bytes(), + mat.public_key_pem.as_bytes(), + mat.kid, + "gateway-a", + Duration::from_mins(15), + ) + .expect("issuer"); let minted = issuer .mint_extension_token( &extension_audience("service-a"), @@ -972,7 +610,7 @@ mod tests { #[test] fn extension_token_rejects_gateway_sandbox_audience() { - let (issuer, _) = pair(); + let issuer = issuer(Duration::from_hours(1)); let error = issuer .mint_extension_token( &extension_audience("openshell-gateway:test-gateway"), @@ -990,7 +628,7 @@ mod tests { #[test] fn extension_token_enforces_positive_bounded_ttl_and_caller_shape() { - let (issuer, _) = pair_with_ttl(None); + let issuer = issuer(Duration::from_mins(15)); for ttl in [ Duration::ZERO, MAX_EXTENSION_TOKEN_TTL + Duration::from_secs(1), @@ -1031,13 +669,15 @@ mod tests { #[test] fn jwks_contains_public_ed25519_key_without_pem_material() { let mat = generate_jwt_key().expect("jwt key"); - let auth = SandboxJwtAuthenticator::from_pem( + let issuer = ExtensionJwtIssuer::from_pem( + mat.signing_key_pem.as_bytes(), mat.public_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", + Duration::from_mins(15), ) - .expect("authenticator"); - let jwks = auth.jwks(); + .expect("issuer"); + let jwks = issuer.jwks(); assert_eq!(jwks.keys.len(), 1); let key = &jwks.keys[0]; assert_eq!(key.kid, mat.kid); diff --git a/crates/openshell-server/src/credentials.rs b/crates/openshell-server/src/credentials.rs index ad1d5c6eb0..901a44f9f3 100644 --- a/crates/openshell-server/src/credentials.rs +++ b/crates/openshell-server/src/credentials.rs @@ -2671,7 +2671,6 @@ socket_path = {socket_path_toml} r#" transport = "in_tree" namespace = "openshell" -allow_reference_namespace = true "#, ), ) @@ -2684,13 +2683,6 @@ allow_reference_namespace = true .and_then(toml::Value::as_str), Some("openshell") ); - assert_eq!( - parsed - .backend_config - .get("allow_reference_namespace") - .and_then(toml::Value::as_bool), - Some(true) - ); assert!(!parsed.backend_config.contains_key("transport")); } diff --git a/crates/openshell-server/src/grpc/auth_rpc.rs b/crates/openshell-server/src/grpc/auth_rpc.rs index ec494e9434..90ebd4a30a 100644 --- a/crates/openshell-server/src/grpc/auth_rpc.rs +++ b/crates/openshell-server/src/grpc/auth_rpc.rs @@ -9,7 +9,7 @@ //! - `RefreshSandboxToken` — renew a still-valid gateway JWT //! //! Both end in a fresh gateway-signed JWT minted by -//! [`crate::auth::sandbox_jwt::SandboxJwtIssuer`]. Refresh atomically advances +//! [`crate::auth::sandbox_jwt::SandboxSessionJwtAuthority`]. Refresh atomically advances //! the sandbox's credential lineage. The immediately consumed bearer may only //! replay the same refresh for a short recovery window; it cannot authorize //! ordinary RPCs or select another successor. @@ -177,7 +177,7 @@ pub async fn handle_refresh_sandbox_token( )); }; - let issuer = state.sandbox_jwt_issuer.as_ref().ok_or_else(|| { + let issuer = state.extension_jwt_issuer.as_ref().ok_or_else(|| { warn!( sandbox_id = %sandbox.sandbox_id, "RefreshSandboxToken called but sandbox JWT issuer is not configured" @@ -319,7 +319,6 @@ pub async fn handle_refresh_sandbox_token( } const MAX_EXTENSION_CREDENTIALS_PER_REFRESH: usize = 64; -const DEFAULT_EXTENSION_TOKEN_TTL: Duration = Duration::from_mins(15); const REFRESH_REPLAY_GRACE: Duration = Duration::from_secs(30); fn current_unix_seconds() -> i64 { @@ -332,7 +331,7 @@ fn current_unix_seconds() -> i64 { #[allow(clippy::result_large_err)] fn mint_extension_credentials( - issuer: &crate::auth::sandbox_jwt::SandboxJwtIssuer, + issuer: &crate::auth::sandbox_jwt::ExtensionJwtIssuer, sandbox_id: &str, requested_names: &[String], available_services: &[openshell_core::proto::SupervisorMiddlewareService], @@ -362,11 +361,7 @@ fn mint_extension_credentials( .iter() .map(|service| (service.name.as_str(), service)) .collect(); - let ttl = issuer - .sandbox_token_ttl() - .map_or(DEFAULT_EXTENSION_TOKEN_TTL, |ttl| { - ttl.min(MAX_EXTENSION_TOKEN_TTL) - }); + let ttl = issuer.token_ttl().min(MAX_EXTENSION_TOKEN_TTL); requested_names .iter() @@ -438,7 +433,7 @@ mod tests { use crate::ServerState; use crate::auth::identity::Identity; use crate::auth::principal::{Principal, SandboxPrincipal, UserPrincipal}; - use crate::auth::sandbox_jwt::{SandboxJwtIssuer, SandboxSessionJwtAuthority}; + use crate::auth::sandbox_jwt::{ExtensionJwtIssuer, SandboxSessionJwtAuthority}; use crate::compute::new_test_runtime; use crate::persistence::Store; use crate::sandbox_index::SandboxIndex; @@ -473,14 +468,15 @@ mod tests { None, ); // We don't need the authenticator for these tests; only the issuer. - let issuer = SandboxJwtIssuer::from_pem( + let issuer = ExtensionJwtIssuer::from_pem( mat.signing_key_pem.as_bytes(), + mat.public_key_pem.as_bytes(), mat.kid.clone(), "test-gateway", - Some(Duration::from_hours(1)), + Duration::from_hours(1), ) .unwrap(); - state.sandbox_jwt_issuer = Some(Arc::new(issuer)); + state.extension_jwt_issuer = Some(Arc::new(issuer)); let authority = Arc::new( SandboxSessionJwtAuthority::from_pem( mat.signing_key_pem.as_bytes(), @@ -710,7 +706,7 @@ mod tests { #[tokio::test] async fn extension_credentials_are_minted_only_for_selected_registration_names() { let state = state_with_issuer().await; - let issuer = state.sandbox_jwt_issuer.as_deref().expect("issuer"); + let issuer = state.extension_jwt_issuer.as_deref().expect("issuer"); let available = vec![openshell_core::proto::SupervisorMiddlewareService { name: "content-guard".to_string(), audience: "urn:example:content-guard".to_string(), @@ -756,7 +752,7 @@ mod tests { #[tokio::test] async fn opted_out_registrations_never_receive_a_minted_credential() { let state = state_with_issuer().await; - let issuer = state.sandbox_jwt_issuer.as_deref().expect("issuer"); + let issuer = state.extension_jwt_issuer.as_deref().expect("issuer"); let available = vec![openshell_core::proto::SupervisorMiddlewareService { name: "legacy-guard".to_string(), audience: "urn:example:legacy-guard".to_string(), @@ -781,7 +777,7 @@ mod tests { #[tokio::test] async fn extension_credential_request_rejects_duplicate_names_atomically() { let state = state_with_issuer().await; - let issuer = state.sandbox_jwt_issuer.as_deref().expect("issuer"); + let issuer = state.extension_jwt_issuer.as_deref().expect("issuer"); let available = vec![openshell_core::proto::SupervisorMiddlewareService { name: "content-guard".to_string(), audience: "urn:example:content-guard".to_string(), diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 4d7c52853d..74fac49de8 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -2813,7 +2813,7 @@ pub(super) async fn load_sandbox_config( policy_source, &supervisor_middleware_services, state.config.policy_validation_failure_mode, - state.sandbox_jwt_issuer.is_some(), + state.extension_jwt_issuer.is_some(), ); if let Some(policy) = policy.as_ref() { validate_policy_credential_binding_context( @@ -2852,7 +2852,7 @@ pub(super) async fn load_sandbox_config( .policy_validation_failure_mode .as_str() .to_string(), - extension_authentication_enabled: state.sandbox_jwt_issuer.is_some(), + extension_authentication_enabled: state.extension_jwt_issuer.is_some(), provider_attachment_epoch: sandbox .spec .as_ref() diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index db98b18bd9..635acd5257 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -629,19 +629,13 @@ async fn handle_create_sandbox_inner( } else { None }; - let sandbox_token = if let Some(authentication) = &launch_authentication { - Some( - authentication - .supervisor - .gateway_token - .expose_secret() - .to_string(), - ) - } else if let Some(issuer) = &state.sandbox_jwt_issuer { - Some(issuer.mint(&id)?.token) - } else { - None - }; + let sandbox_token = launch_authentication.as_ref().map(|authentication| { + authentication + .supervisor + .gateway_token + .expose_secret() + .to_string() + }); let launch_authentication = launch_authentication .map(|authentication| { serde_json::to_vec(&authentication) diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 5a43364247..ff5c160be5 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -100,12 +100,8 @@ struct GatewayExtensionCredential { ttl: Duration, } -fn extension_token_ttl(issuer: &auth::sandbox_jwt::SandboxJwtIssuer) -> Duration { - issuer - .sandbox_token_ttl() - .map_or(Duration::from_mins(15), |ttl| { - ttl.min(MAX_EXTENSION_TOKEN_TTL) - }) +fn extension_token_ttl(issuer: &auth::sandbox_jwt::ExtensionJwtIssuer) -> Duration { + issuer.token_ttl().min(MAX_EXTENSION_TOKEN_TTL) } /// Mint the gateway-caller credential for one extension registration. @@ -115,7 +111,7 @@ fn extension_token_ttl(issuer: &auth::sandbox_jwt::SandboxJwtIssuer) -> Duration /// downgrades a security boundary, so it is reported once per registration at /// startup rather than being silently tolerated. fn mint_gateway_extension_credential( - issuer: &Arc, + issuer: &Arc, kind: ExtensionKind, name: &str, audience: &str, @@ -182,7 +178,7 @@ fn mint_gateway_extension_credential( } fn spawn_gateway_extension_token_refresh( - issuer: Arc, + issuer: Arc, credentials: Vec, ) { if credentials.is_empty() { @@ -326,20 +322,12 @@ pub struct ServerState { /// OIDC JWKS cache for JWT validation. `None` when OIDC is not configured. pub oidc_cache: Option>, - /// Gateway-minted sandbox JWT issuer. `None` when `config.gateway_jwt` - /// is not configured; in that mode `IssueSandboxToken` returns - /// `Status::unavailable`. Populated at startup from the on-disk key - /// material that `certgen` writes. - pub sandbox_jwt_issuer: Option>, + /// Typed extension JWT issuer and public verification metadata. + pub extension_jwt_issuer: Option>, /// Launch-scoped gateway and Sandbox Protocol token authority. pub sandbox_session_jwt_authority: Option>, - /// Authenticator that validates gateway-minted sandbox JWTs on every - /// inbound request. Always set when `sandbox_jwt_issuer` is, so callers - /// presenting a freshly minted token are recognized. - pub sandbox_jwt_authenticator: Option>, - /// Optional selected-driver authenticator for the `IssueSandboxToken` /// bootstrap path. pub compute_driver_authenticator: Option>, @@ -451,9 +439,8 @@ impl ServerState { extension_mint_limiter: auth::extension_mint_limit::ExtensionMintLimiter::default(), middleware_registry: Arc::new(MiddlewareRegistry::default()), oidc_cache, - sandbox_jwt_issuer: None, + extension_jwt_issuer: None, sandbox_session_jwt_authority: None, - sandbox_jwt_authenticator: None, compute_driver_authenticator: None, peer_authenticator: None, grpc_rate_limiter, @@ -545,7 +532,7 @@ pub(crate) async fn run_server( // Load signing material before connecting remote extensions so their // startup Describe calls can authenticate with gateway-caller tokens. - let (sandbox_jwt_issuer, sandbox_jwt_authenticator, sandbox_session_jwt_authority) = + let (extension_jwt_issuer, sandbox_session_jwt_authority) = if let Some(ref jwt) = config.gateway_jwt { let signing_pem = std::fs::read(&jwt.signing_key_path).map_err(|e| { Error::config(format!( @@ -575,19 +562,12 @@ pub(crate) async fn run_server( ))); } let issuer = Arc::new( - auth::sandbox_jwt::SandboxJwtIssuer::from_pem( + auth::sandbox_jwt::ExtensionJwtIssuer::from_pem( &signing_pem, - kid.clone(), - &jwt.gateway_id, - jwt.sandbox_token_ttl(), - ) - .map_err(Error::config)?, - ); - let authenticator = Arc::new( - auth::sandbox_jwt::SandboxJwtAuthenticator::from_pem( &public_pem, kid.clone(), &jwt.gateway_id, + jwt.token_ttl(), ) .map_err(Error::config)?, ); @@ -597,7 +577,7 @@ pub(crate) async fn run_server( &public_pem, kid, &jwt.gateway_id, - jwt.sandbox_token_ttl().unwrap_or(Duration::from_mins(15)), + jwt.token_ttl(), ) .map_err(Error::config)?, ); @@ -606,9 +586,9 @@ pub(crate) async fn run_server( ttl_secs = jwt.ttl_secs.map(std::num::NonZeroU64::get), "gateway-minted sandbox JWT enabled" ); - (Some(issuer), Some(authenticator), Some(session_authority)) + (Some(issuer), Some(session_authority)) } else { - (None, None, None) + (None, None) }; let middleware_registrations = config_file @@ -626,7 +606,7 @@ pub(crate) async fn run_server( .unwrap_or_default(); let mut gateway_extension_credentials = Vec::new(); let middleware_registry = Arc::new( - if let Some(issuer) = sandbox_jwt_issuer.as_ref() { + if let Some(issuer) = extension_jwt_issuer.as_ref() { let mut slots = HashMap::new(); for registration in &middleware_registrations { if let Some(credential) = mint_gateway_extension_credential( @@ -706,7 +686,7 @@ pub(crate) async fn run_server( shutdown_rx.clone(), ) .await?; - let gateway_interceptors = if let Some(issuer) = sandbox_jwt_issuer.as_ref() { + let gateway_interceptors = if let Some(issuer) = extension_jwt_issuer.as_ref() { let mut slots = BTreeMap::new(); for interceptor in &config.gateway_interceptors { let audience = interceptor.resolved_audience(); @@ -758,14 +738,15 @@ pub(crate) async fn run_server( state.middleware_registry = middleware_registry; state.gateway_interceptors = gateway_interceptors; state.provider_profile_sources = provider_profile_sources; - state.sandbox_jwt_issuer = sandbox_jwt_issuer.clone(); - state.sandbox_jwt_authenticator = sandbox_jwt_authenticator; + state.extension_jwt_issuer = extension_jwt_issuer.clone(); state.sandbox_session_jwt_authority = sandbox_session_jwt_authority; - if let Some(issuer) = sandbox_jwt_issuer { + if let Some(issuer) = extension_jwt_issuer { spawn_gateway_extension_token_refresh(issuer, gateway_extension_credentials); } - if state.sandbox_jwt_issuer.is_some() && state.compute.supports_sandbox_authentication() { + if state.sandbox_session_jwt_authority.is_some() + && state.compute.supports_sandbox_authentication() + { state.compute_driver_authenticator = Some(Arc::new( auth::compute_driver::ComputeDriverAuthenticator::new(state.compute.clone()), )); @@ -1676,17 +1657,6 @@ async fn build_compute_runtime( let admission = compute::driver_config::admission_config_from_context(driver_startup, driver.name())?; info!(driver = %driver.name(), "Using compute driver"); - if config - .gateway_jwt - .as_ref() - .is_some_and(|jwt| jwt.sandbox_token_ttl().is_none()) - && !driver.is_local_singleplayer(registry) - { - warn!( - "Gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs is omitted); set gateway_jwt.ttl_secs > 0 for shared deployments" - ); - } - let runtime = match driver { ConfiguredComputeDriver::Registered(registration) => { let build_context = ComputeDriverBuildContext { @@ -1776,15 +1746,6 @@ impl ConfiguredComputeDriver { } } - fn is_local_singleplayer(&self, registry: &ComputeDriverRegistry) -> bool { - match self { - Self::Registered(registration) => registration.is_local_singleplayer(), - Self::Remote { name } => registry - .get(name) - .is_some_and(ComputeDriverRegistration::is_local_singleplayer), - } - } - fn telemetry_compute_driver(&self, registry: &ComputeDriverRegistry) -> TelemetryComputeDriver { match self { Self::Registered(registration) => registration.telemetry_category, @@ -2007,17 +1968,18 @@ mod tests { record_detection_probe("third", true) } - fn extension_test_issuer() -> Arc { - extension_test_issuer_with_ttl(Some(Duration::from_mins(15))) + fn extension_test_issuer() -> Arc { + extension_test_issuer_with_ttl(Duration::from_mins(15)) } fn extension_test_issuer_with_ttl( - ttl: Option, - ) -> Arc { + ttl: Duration, + ) -> Arc { let material = openshell_bootstrap::jwt::generate_jwt_key().expect("jwt key"); Arc::new( - crate::auth::sandbox_jwt::SandboxJwtIssuer::from_pem( + crate::auth::sandbox_jwt::ExtensionJwtIssuer::from_pem( material.signing_key_pem.as_bytes(), + material.public_key_pem.as_bytes(), material.kid, "gateway-a", ttl, @@ -2027,17 +1989,17 @@ mod tests { } #[test] - fn non_expiring_sandbox_tokens_use_finite_extension_ttl() { - let issuer = extension_test_issuer_with_ttl(None); + fn gateway_ttl_bounds_extension_ttl() { + let issuer = extension_test_issuer_with_ttl(Duration::from_mins(15)); assert_eq!(extension_token_ttl(&issuer), Duration::from_mins(15)); } #[test] fn extension_token_ttl_is_capped_at_one_hour() { - let issuer = extension_test_issuer_with_ttl(Some(Duration::from_hours(24))); + let issuer = extension_test_issuer_with_ttl(Duration::from_hours(24)); assert_eq!(extension_token_ttl(&issuer), Duration::from_hours(1)); - let short = extension_test_issuer_with_ttl(Some(Duration::from_mins(5))); + let short = extension_test_issuer_with_ttl(Duration::from_mins(5)); assert_eq!(extension_token_ttl(&short), Duration::from_mins(5)); } diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 45630a05f7..2dfaa4ede3 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -867,12 +867,9 @@ where /// — delegates a driver-native credential and receives a sandbox identity /// so the handler can mint a gateway JWT. No-op on every other path. /// 3. `SandboxSessionJwtAuthenticator` — validates generation-bound gateway -/// JWTs against the durable sandbox identity. When configured, legacy -/// unbound sandbox JWTs are deliberately not admitted. -/// 4. `SandboxJwtAuthenticator` — legacy fallback used only when session JWT -/// authentication is unavailable. Recognized via a distinctive `kid` so -/// non-matching Bearer tokens fall through. -/// 5. `OidcAuthenticator` — validates user Bearer tokens against the +/// JWTs against the durable sandbox identity. Untyped legacy sandbox JWTs +/// are not admitted. +/// 4. `OidcAuthenticator` — validates user Bearer tokens against the /// configured OIDC issuer. Returns `Unauthenticated` for missing /// Bearer headers so non-OIDC clients can't sneak through. /// @@ -893,7 +890,6 @@ fn build_authenticator_chain(state: &ServerState) -> Option if let Some(driver) = state.compute_driver_authenticator.clone() { authenticators.push(driver); } - let session_authentication_enabled = state.sandbox_session_jwt_authority.is_some(); if let Some(authority) = state.sandbox_session_jwt_authority.clone() { authenticators.push(Arc::new( crate::auth::sandbox_jwt::SandboxSessionJwtAuthenticator::new( @@ -902,9 +898,6 @@ fn build_authenticator_chain(state: &ServerState) -> Option ), )); } - if !session_authentication_enabled && let Some(jwt) = state.sandbox_jwt_authenticator.clone() { - authenticators.push(jwt); - } if let Some(cache) = state.oidc_cache.clone() { authenticators.push(Arc::new(OidcAuthenticator::new(cache))); } @@ -2546,6 +2539,8 @@ mod tests { Principal, SandboxIdentitySource, SandboxPrincipal, UserPrincipal, }; use http_body_util::Full; + use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; + use serde::Serialize; use std::sync::Arc; use std::sync::Mutex; use tower::Service; @@ -2645,6 +2640,36 @@ mod tests { }) } + #[derive(Serialize)] + struct LegacySandboxClaims { + sub: String, + iss: String, + aud: String, + iat: i64, + exp: i64, + sandbox_id: String, + } + + fn legacy_sandbox_token(material: &openshell_bootstrap::jwt::JwtKeyMaterial) -> String { + let claims = LegacySandboxClaims { + sub: "spiffe://openshell/sandbox/sandbox-a".to_string(), + iss: "openshell-gateway:test".to_string(), + aud: "openshell-gateway:test".to_string(), + iat: 1, + exp: 0, + sandbox_id: "sandbox-a".to_string(), + }; + let mut header = Header::new(Algorithm::EdDSA); + header.kid = Some(material.kid.clone()); + encode( + &header, + &claims, + &EncodingKey::from_ed_pem(material.signing_key_pem.as_bytes()) + .expect("signing key"), + ) + .expect("legacy token") + } + #[tokio::test] async fn mtls_peer_identity_fills_missing_principal_when_enabled() { let mock = Arc::new(MockAuthenticator::returning(Ok(None))); @@ -2929,6 +2954,52 @@ mod tests { assert_eq!(grpc_status(&res).as_deref(), Some("16")); } + #[tokio::test] + async fn legacy_sandbox_jwt_cannot_reach_sensitive_sandbox_methods() { + let material = openshell_bootstrap::jwt::generate_jwt_key().expect("JWT key"); + let authority = Arc::new( + crate::auth::sandbox_jwt::SandboxSessionJwtAuthority::from_pem( + material.signing_key_pem.as_bytes(), + material.public_key_pem.as_bytes(), + material.kid.clone(), + "test", + Duration::from_mins(15), + ) + .expect("session authority"), + ); + let store = Arc::new( + crate::persistence::Store::connect("sqlite::memory:") + .await + .expect("store"), + ); + let token = legacy_sandbox_token(&material); + + for path in [ + "/openshell.v1.OpenShell/GetSandboxProviderEnvironment", + "/openshell.v1.OpenShell/ConnectSupervisor", + ] { + let authenticator = Arc::new( + crate::auth::sandbox_jwt::SandboxSessionJwtAuthenticator::new( + authority.clone(), + store.clone(), + ), + ); + let chain = AuthenticatorChain::new(vec![authenticator]); + let (recorder, seen) = PrincipalRecorder::new(); + let mut router = AuthGrpcRouter::new(recorder, Some(chain), None); + let request = Request::builder() + .uri(path) + .header("authorization", format!("Bearer {token}")) + .body(Full::new(Bytes::new())) + .expect("request"); + + let response = router.call(request).await.expect("router response"); + + assert!(seen.lock().unwrap().is_none(), "{path} reached handler"); + assert_eq!(grpc_status(&response).as_deref(), Some("16"), "{path}"); + } + } + #[tokio::test] async fn authenticator_error_short_circuits() { let mock = Arc::new(MockAuthenticator::returning(Err( diff --git a/crates/openshell-supervisor-network/src/host.rs b/crates/openshell-supervisor-network/src/host.rs index f52ec29fb8..7e29b3d0ed 100644 --- a/crates/openshell-supervisor-network/src/host.rs +++ b/crates/openshell-supervisor-network/src/host.rs @@ -21,7 +21,7 @@ use openshell_core::policy::ProxyPolicy; use openshell_core::proposals::AgentProposals; use openshell_core::proto::SandboxPolicy as ProtoSandboxPolicy; use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_isolation_interface::contract::{BinaryIdentity, Sha256Digest}; +use openshell_isolation_interface::contract::{BinaryIdentity, ExecutableIdentity, Sha256Digest}; use openshell_ocsf::{ ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ctx::ctx as ocsf_ctx, ocsf_emit, }; @@ -197,8 +197,10 @@ pub async fn start_host_proxy(config: HostProxyConfig) -> Result, } /// Thread-safe cache of binary SHA256 hashes for TOFU enforcement. @@ -96,6 +99,16 @@ pub struct BinaryIdentityCache { hashes: Mutex>, } +#[derive(Debug, thiserror::Error)] +pub(crate) enum SuppliedIdentityError { + #[error("{0}")] + Unavailable(String), + #[error( + "Binary identity cache capacity exhausted (maximum {MAX_IDENTITY_CACHE_ENTRIES} pinned paths)" + )] + CapacityExhausted, +} + impl Default for BinaryIdentityCache { fn default() -> Self { Self::new() @@ -120,6 +133,68 @@ impl BinaryIdentityCache { self.verify_or_cache_with_paths(path, path, procfs::file_sha256) } + /// Atomically verify or pin every authorization-capable executable in a + /// backend-supplied identity chain. + pub(crate) fn verify_or_cache_supplied_identity( + &self, + identity: &BinaryIdentity, + ) -> std::result::Result<(), SuppliedIdentityError> { + let mut supplied = HashMap::new(); + for executable in std::iter::once(&identity.executable).chain(&identity.ancestors) { + if executable.path.as_os_str().is_empty() || !executable.path.is_absolute() { + return Err(SuppliedIdentityError::Unavailable(format!( + "Invalid executable identity path: {} must be absolute", + executable.path.display() + ))); + } + let digest = executable.digest.ok_or_else(|| { + SuppliedIdentityError::Unavailable(format!( + "Invalid executable identity evidence: {} has missing digest", + executable.path.display() + )) + })?; + if let Some(existing) = supplied.insert(executable.path.clone(), digest) + && existing != digest + { + return Err(SuppliedIdentityError::Unavailable(format!( + "Invalid executable identity: conflicting evidence for {}", + executable.path.display() + ))); + } + } + + let mut hashes = self.hashes.lock().map_err(|_| { + SuppliedIdentityError::Unavailable("Binary identity cache lock poisoned".to_string()) + })?; + + for (path, digest) in &supplied { + if let Some(existing) = hashes.get(path) + && existing.hash != digest.to_string() + { + return Err(SuppliedIdentityError::Unavailable(format!( + "Binary integrity violation: {} executable changed", + path.display() + ))); + } + } + + let new_entry_count = supplied + .keys() + .filter(|path| !hashes.contains_key(*path)) + .count(); + if new_entry_count > MAX_IDENTITY_CACHE_ENTRIES.saturating_sub(hashes.len()) { + return Err(SuppliedIdentityError::CapacityExhausted); + } + + for (path, digest) in supplied { + hashes.entry(path).or_insert_with(|| CachedBinary { + hash: digest.to_string(), + fingerprint: None, + }); + } + Ok(()) + } + #[cfg(target_os = "linux")] pub fn verify_or_cache_process_exe(&self, display_path: &Path, pid: u32) -> Result { let proc_exe = PathBuf::from(format!("/proc/{pid}/exe")); @@ -148,7 +223,7 @@ impl BinaryIdentityCache { .cloned(); if let Some(cached_binary) = &cached - && cached_binary.fingerprint == fingerprint + && cached_binary.fingerprint.as_ref() == Some(&fingerprint) { debug!( " verify_or_cache: {}ms CACHE HIT path={}", @@ -175,10 +250,14 @@ impl BinaryIdentityCache { && existing.hash != current_hash { return Err(miette::miette!( - "Binary integrity violation: {} hash changed (cached: {}, current: {})", - cache_path.display(), - existing.hash, - current_hash + "Binary integrity violation: {} executable changed", + cache_path.display() + )); + } + + if !hashes.contains_key(cache_path) && hashes.len() >= MAX_IDENTITY_CACHE_ENTRIES { + return Err(miette::miette!( + "Binary identity cache capacity exhausted (maximum {MAX_IDENTITY_CACHE_ENTRIES} pinned paths)" )); } @@ -186,7 +265,7 @@ impl BinaryIdentityCache { cache_path.to_path_buf(), CachedBinary { hash: current_hash.clone(), - fingerprint, + fingerprint: Some(fingerprint), }, ); @@ -204,9 +283,288 @@ impl BinaryIdentityCache { mod tests { use super::*; use crate::procfs; + use openshell_isolation_interface::contract::{ + BinaryIdentity, ExecutableIdentity, Sha256Digest, + }; use std::io::Write; use std::time::Duration; + fn digest(value: &str) -> Sha256Digest { + value.repeat(32).parse().unwrap() + } + + fn supplied_identity( + executable_path: &str, + executable_digest: Option<&str>, + ancestors: &[(&str, Option<&str>)], + ) -> BinaryIdentity { + BinaryIdentity { + executable: ExecutableIdentity { + path: PathBuf::from(executable_path), + digest: executable_digest.map(digest), + }, + ancestors: ancestors + .iter() + .map(|(path, digest_value)| ExecutableIdentity { + path: PathBuf::from(path), + digest: digest_value.map(digest), + }) + .collect(), + cmdline_paths: Vec::new(), + } + } + + #[test] + fn supplied_identity_reuses_pin_and_rejects_leaf_replacement() { + let cache = BinaryIdentityCache::new(); + let original = supplied_identity("/sandbox/tool", Some("11"), &[]); + let replaced = supplied_identity("/sandbox/tool", Some("22"), &[]); + + cache.verify_or_cache_supplied_identity(&original).unwrap(); + cache.verify_or_cache_supplied_identity(&original).unwrap(); + let error = cache + .verify_or_cache_supplied_identity(&replaced) + .unwrap_err() + .to_string(); + + assert!(error.contains("/sandbox/tool")); + assert!(error.contains("integrity violation")); + assert!(!error.contains(&"11".repeat(32))); + assert!(!error.contains(&"22".repeat(32))); + } + + #[test] + fn supplied_identity_rejects_ancestor_replacement() { + let cache = BinaryIdentityCache::new(); + cache + .verify_or_cache_supplied_identity(&supplied_identity( + "/sandbox/tool", + Some("11"), + &[("/sandbox/launcher", Some("22"))], + )) + .unwrap(); + + let error = cache + .verify_or_cache_supplied_identity(&supplied_identity( + "/sandbox/tool", + Some("11"), + &[("/sandbox/launcher", Some("33"))], + )) + .unwrap_err() + .to_string(); + + assert!(error.contains("/sandbox/launcher")); + assert!(!error.contains(&"22".repeat(32))); + assert!(!error.contains(&"33".repeat(32))); + } + + #[test] + fn supplied_identity_rejects_missing_evidence() { + let cache = BinaryIdentityCache::new(); + let missing_leaf = supplied_identity("/sandbox/tool", None, &[]); + let missing_ancestor = + supplied_identity("/sandbox/tool", Some("11"), &[("/sandbox/launcher", None)]); + + let leaf_error = cache + .verify_or_cache_supplied_identity(&missing_leaf) + .unwrap_err() + .to_string(); + let ancestor_error = cache + .verify_or_cache_supplied_identity(&missing_ancestor) + .unwrap_err() + .to_string(); + + assert!(leaf_error.contains("/sandbox/tool")); + assert!(leaf_error.contains("missing digest")); + assert!(ancestor_error.contains("/sandbox/launcher")); + assert!(ancestor_error.contains("missing digest")); + } + + #[test] + fn supplied_identity_rejects_non_absolute_or_empty_paths() { + let cache = BinaryIdentityCache::new(); + for path in ["", "sandbox/tool"] { + let error = cache + .verify_or_cache_supplied_identity(&supplied_identity(path, Some("11"), &[])) + .unwrap_err() + .to_string(); + assert!(error.contains("must be absolute")); + } + assert!(cache.hashes.lock().unwrap().is_empty()); + } + + #[test] + fn supplied_identity_deduplicates_matching_paths() { + let cache = BinaryIdentityCache::new(); + let identity = supplied_identity( + "/sandbox/tool", + Some("11"), + &[("/sandbox/tool", Some("11"))], + ); + + cache.verify_or_cache_supplied_identity(&identity).unwrap(); + + assert_eq!(cache.hashes.lock().unwrap().len(), 1); + } + + #[test] + fn supplied_identity_conflicting_duplicate_is_atomic() { + let cache = BinaryIdentityCache::new(); + let identity = supplied_identity( + "/sandbox/tool", + Some("11"), + &[ + ("/sandbox/new-sibling", Some("33")), + ("/sandbox/tool", Some("22")), + ], + ); + + let error = cache + .verify_or_cache_supplied_identity(&identity) + .unwrap_err() + .to_string(); + + assert!(error.contains("/sandbox/tool")); + assert!(error.contains("conflicting evidence")); + assert!(cache.hashes.lock().unwrap().is_empty()); + } + + #[test] + fn supplied_identity_conflict_with_existing_pin_is_atomic() { + let cache = BinaryIdentityCache::new(); + cache + .verify_or_cache_supplied_identity(&supplied_identity("/sandbox/tool", Some("11"), &[])) + .unwrap(); + let conflicting_chain = supplied_identity( + "/sandbox/other", + Some("33"), + &[("/sandbox/tool", Some("22"))], + ); + + cache + .verify_or_cache_supplied_identity(&conflicting_chain) + .unwrap_err(); + + assert!( + !cache + .hashes + .lock() + .unwrap() + .contains_key(Path::new("/sandbox/other")) + ); + } + + #[test] + fn supplied_identity_capacity_rejection_is_atomic() { + let cache = BinaryIdentityCache::new(); + for index in 0..4095 { + cache + .verify_or_cache_supplied_identity(&supplied_identity( + &format!("/sandbox/pinned-{index}"), + Some("11"), + &[], + )) + .unwrap(); + } + let overflowing_chain = supplied_identity( + "/sandbox/new-leaf", + Some("22"), + &[("/sandbox/new-ancestor", Some("33"))], + ); + + let error = cache + .verify_or_cache_supplied_identity(&overflowing_chain) + .unwrap_err() + .to_string(); + + assert!(error.contains("capacity")); + let hashes = cache.hashes.lock().unwrap(); + assert_eq!(hashes.len(), 4095); + assert!(!hashes.contains_key(Path::new("/sandbox/new-leaf"))); + assert!(!hashes.contains_key(Path::new("/sandbox/new-ancestor"))); + } + + #[test] + fn legacy_identity_observation_respects_cache_capacity() { + let executable = tempfile::NamedTempFile::new().unwrap(); + let cache = BinaryIdentityCache::new(); + for index in 0..4096 { + cache + .verify_or_cache_with_paths( + &PathBuf::from(format!("/sandbox/pinned-{index}")), + executable.path(), + |_| Ok("11".repeat(32)), + ) + .unwrap(); + } + + let error = cache + .verify_or_cache_with_paths(Path::new("/sandbox/overflow"), executable.path(), |_| { + Ok("11".repeat(32)) + }) + .unwrap_err() + .to_string(); + + assert!(error.contains("capacity")); + assert_eq!(cache.hashes.lock().unwrap().len(), 4096); + } + + #[test] + fn supplied_identity_shares_pin_with_legacy_observation() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(b"binary content").unwrap(); + tmp.flush().unwrap(); + let path = tmp.path(); + let cache = BinaryIdentityCache::new(); + let legacy_digest = cache.verify_or_cache(path).unwrap(); + let same = BinaryIdentity { + executable: ExecutableIdentity { + path: path.to_path_buf(), + digest: Some(legacy_digest.parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }; + let different = supplied_identity(path.to_str().unwrap(), Some("11"), &[]); + + cache.verify_or_cache_supplied_identity(&same).unwrap(); + cache + .verify_or_cache_supplied_identity(&different) + .unwrap_err(); + } + + #[test] + fn legacy_observation_validates_and_attaches_to_supplied_pin() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("binary"); + std::fs::write(&path, b"binary content").unwrap(); + let live_digest = procfs::file_sha256(&path).unwrap(); + let supplied = BinaryIdentity { + executable: ExecutableIdentity { + path: path.clone(), + digest: Some(live_digest.parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }; + let cache = BinaryIdentityCache::new(); + + cache.verify_or_cache_supplied_identity(&supplied).unwrap(); + assert_eq!(cache.verify_or_cache(&path).unwrap(), live_digest); + + let original_mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, b"replaced content").unwrap(); + let bumped_mtime = original_mtime.checked_add(Duration::from_secs(2)).unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(bumped_mtime) + .unwrap(); + + cache.verify_or_cache(&path).unwrap_err(); + } + #[test] fn first_call_caches_hash() { let mut tmp = tempfile::NamedTempFile::new().unwrap(); diff --git a/crates/openshell-supervisor-network/src/identity_source.rs b/crates/openshell-supervisor-network/src/identity_source.rs index f5446ddbda..f1b5445902 100644 --- a/crates/openshell-supervisor-network/src/identity_source.rs +++ b/crates/openshell-supervisor-network/src/identity_source.rs @@ -86,8 +86,7 @@ impl ProcfsIdentityResolver { return Err(ResolveError::NotFound); }; if identities.iter().skip(1).any(|candidate| { - candidate.binary_path != identity.binary_path - || candidate.binary_digest != identity.binary_digest + candidate.executable != identity.executable || candidate.ancestors != identity.ancestors || candidate.cmdline_paths != identity.cmdline_paths }) { @@ -106,7 +105,7 @@ mod tests { /// Stands in for the mediation service: a binary-scoped rule can only be /// authorized by a resolved identity carrying the fields it requires. fn admits_binary_rule(result: Result) -> bool { - matches!(result, Ok(identity) if identity.binary_digest.is_some()) + matches!(result, Ok(identity) if identity.executable.digest.is_some()) } #[test] diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index d630354754..b7f3e3b920 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -7,7 +7,7 @@ pub(crate) mod destination; mod egress; mod relay; -use crate::identity::BinaryIdentityCache; +use crate::identity::{BinaryIdentityCache, SuppliedIdentityError}; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; #[cfg(target_os = "linux")] @@ -30,6 +30,8 @@ use openshell_core::net::{ use openshell_core::policy::ProxyPolicy; use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; +#[cfg(test)] +use openshell_isolation_interface::contract::ExecutableIdentity as ContractExecutableIdentity; use openshell_isolation_interface::contract::{ BinaryIdentity as ContractBinaryIdentity, BoundaryDuplexStream, MediationTiming, NetworkMediationSource, PendingTcpOpen, ResolveError, TcpOpenDecision, TcpOpenDenial, @@ -430,6 +432,7 @@ impl ProxyHandle { let tx = preauthorized_tx.clone(); let dns_store = policy_dns_store.clone(); let opa = opa_engine.clone(); + let cache = identity_cache.clone(); let backend_gateway = *backend_host_gateway; let trusted_gateway = *trusted_host_gateway; tokio::spawn(async move { @@ -437,6 +440,7 @@ impl ProxyHandle { connection, dns_store.as_ref(), &opa, + &cache, backend_gateway, trusted_gateway, ) @@ -582,6 +586,7 @@ async fn preauthorize_transparent_open( connection: PendingTcpOpen, policy_dns_store: Option<&Arc>, opa_engine: &OpaEngine, + identity_cache: &BinaryIdentityCache, backend_host_gateway: Option, trusted_host_gateway: Option, ) -> Option { @@ -612,24 +617,29 @@ async fn preauthorize_transparent_open( return None; } }; - let mut decision = authorize_supplied_identity( + let supplied_authorization = authorize_supplied_identity_with_denial( opa_engine, + identity_cache, EgressIntent::connect(host.clone(), destination.port()), &binary_identity, ); + let mut decision = supplied_authorization.decision; if let NetworkAction::Deny { reason } = &decision.action { - warn!(%destination, %reason, "Denied staged transparent connection"); - emit_staged_transparent_denial( - destination, - &binary_identity, - reason, - "transparent_tcp_policy_denied", + let (denial, status_detail) = supplied_authorization.denial.map_or( + (TcpOpenDenial::PolicyDenied, "transparent_tcp_policy_denied"), + |denial| match denial { + SuppliedIdentityDenial::IdentityUnavailable => ( + TcpOpenDenial::IdentityUnavailable, + "transparent_tcp_identity_unavailable", + ), + SuppliedIdentityDenial::ResourceExhausted => ( + TcpOpenDenial::ResourceExhausted, + "transparent_tcp_resource_exhausted", + ), + }, ); - let denial = if binary_identity.is_err() { - TcpOpenDenial::IdentityUnavailable - } else { - TcpOpenDenial::PolicyDenied - }; + warn!(%destination, %reason, "Denied staged transparent connection"); + emit_staged_transparent_denial(destination, &binary_identity, reason, status_detail); let _ = completion.send(TcpOpenDecision::Denied(denial)); return None; } @@ -718,11 +728,11 @@ fn emit_staged_transparent_denial( |_| ("-".to_string(), "-".to_string(), "-".to_string()), |identity| { ( - identity.binary_path.display().to_string(), + identity.executable.path.display().to_string(), identity .ancestors .iter() - .map(|path| path.display().to_string()) + .map(|ancestor| ancestor.path.display().to_string()) .collect::>() .join(" -> "), identity @@ -2314,7 +2324,7 @@ async fn handle_mediated_connection( let mut decision = if let Some(decision) = preauthorized_decision.take() { decision } else if let Some(identity) = supplied_identity.as_ref() { - authorize_supplied_identity(&opa_engine, intent, identity) + authorize_supplied_identity(&opa_engine, &identity_cache, intent, identity) } else if !opa_engine.binary_identity_required() { evaluate_endpoint_only_opa(&opa_engine, intent) } else { @@ -3344,9 +3354,30 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres /// listeners continue to resolve through procfs in `authorize_egress_intent`. fn authorize_supplied_identity( engine: &OpaEngine, + identity_cache: &BinaryIdentityCache, intent: EgressIntent, identity: &Result, ) -> EgressDecision { + authorize_supplied_identity_with_denial(engine, identity_cache, intent, identity).decision +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SuppliedIdentityDenial { + IdentityUnavailable, + ResourceExhausted, +} + +struct SuppliedIdentityAuthorization { + decision: EgressDecision, + denial: Option, +} + +fn authorize_supplied_identity_with_denial( + engine: &OpaEngine, + identity_cache: &BinaryIdentityCache, + intent: EgressIntent, + identity: &Result, +) -> SuppliedIdentityAuthorization { let deny = |reason: String, binary: Option, ancestors: Vec, @@ -3365,48 +3396,71 @@ fn authorize_supplied_identity( let identity = match identity { Ok(identity) => identity, Err(error) => { - return deny( - format!("backend identity resolution failed: {error}"), - None, - vec![], - vec![], - ); + return SuppliedIdentityAuthorization { + decision: deny( + format!("backend identity resolution failed: {error}"), + None, + vec![], + vec![], + ), + denial: Some(SuppliedIdentityDenial::IdentityUnavailable), + }; } }; - let Some(digest) = identity.binary_digest else { - return deny( - "backend identity did not include the required binary digest".to_string(), - Some(identity.binary_path.clone()), - identity.ancestors.clone(), - identity.cmdline_paths.clone(), - ); - }; + let ancestor_paths = identity + .ancestors + .iter() + .map(|ancestor| ancestor.path.clone()) + .collect::>(); + if let Err(error) = identity_cache.verify_or_cache_supplied_identity(identity) { + let denial = match &error { + SuppliedIdentityError::Unavailable(_) => SuppliedIdentityDenial::IdentityUnavailable, + SuppliedIdentityError::CapacityExhausted => SuppliedIdentityDenial::ResourceExhausted, + }; + return SuppliedIdentityAuthorization { + decision: deny( + error.to_string(), + Some(identity.executable.path.clone()), + ancestor_paths, + identity.cmdline_paths.clone(), + ), + denial: Some(denial), + }; + } + let digest = identity + .executable + .digest + .expect("supplied identity validation requires a leaf digest"); let input = crate::opa::NetworkInput { host: intent.destination.host.clone(), port: intent.destination.port, - binary_path: identity.binary_path.clone(), + binary_path: identity.executable.path.clone(), binary_sha256: digest.to_string(), - ancestors: identity.ancestors.clone(), + ancestors: ancestor_paths.clone(), cmdline_paths: identity.cmdline_paths.clone(), }; - match engine.authorize_egress(&input) { + let decision = match engine.authorize_egress(&input) { Ok(authorization) => EgressDecision { intent, action: authorization.action.clone(), policy_generation: authorization.generation, identity: ProcessIdentityEvidence::Available, endpoint: EndpointDecision::from_authorization(&authorization), - binary: Some(identity.binary_path.clone()), + binary: Some(identity.executable.path.clone()), binary_pid: None, - ancestors: identity.ancestors.clone(), + ancestors: ancestor_paths, cmdline_paths: identity.cmdline_paths.clone(), }, Err(error) => deny( format!("policy evaluation error: {error}"), - Some(identity.binary_path.clone()), - identity.ancestors.clone(), + Some(identity.executable.path.clone()), + ancestor_paths, identity.cmdline_paths.clone(), ), + }; + SuppliedIdentityAuthorization { + decision, + denial: None, } } @@ -4999,7 +5053,7 @@ async fn handle_forward_proxy( // 2. Evaluate OPA policy (same identity binding as CONNECT) let intent = EgressIntent::forward_http(host_lc.clone(), port); let mut decision = if let Some(identity) = supplied_identity { - authorize_supplied_identity(&opa_engine, intent, identity) + authorize_supplied_identity(&opa_engine, &identity_cache, intent, identity) } else if !opa_engine.binary_identity_required() { evaluate_endpoint_only_opa(&opa_engine, intent) } else { @@ -6572,14 +6626,17 @@ process: ) .expect("load policy"); let identity = Ok(ContractBinaryIdentity { - binary_path: PathBuf::from("/usr/bin/python3"), - binary_digest: Some("00".repeat(32).parse().expect("digest")), + executable: ContractExecutableIdentity { + path: PathBuf::from("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().expect("digest")), + }, ancestors: Vec::new(), cmdline_paths: Vec::new(), }); let mut decision = authorize_supplied_identity( &engine, + &BinaryIdentityCache::new(), EgressIntent::connect("api.example.com".to_string(), 443), &identity, ); @@ -6594,6 +6651,219 @@ process: assert!(route.configs[0].config.request_body_credential_rewrite); } + #[test] + fn supplied_identity_rejects_same_path_replacement() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + credentialed: + name: credentialed + endpoints: + - host: api.example.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: { method: GET, path: /allowed } + binaries: + - path: /sandbox/bin/client +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .expect("load policy"); + let identity = |digest_byte: &str| { + Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: PathBuf::from("/sandbox/bin/client"), + digest: Some(digest_byte.repeat(32).parse().expect("digest")), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }) + }; + let intent = || EgressIntent::connect("api.example.com".to_string(), 443); + let identity_cache = BinaryIdentityCache::new(); + + let original = + authorize_supplied_identity(&engine, &identity_cache, intent(), &identity("11")); + assert!( + matches!(original.action, NetworkAction::Allow { .. }), + "the policy should authorize the original executable" + ); + + let replacement = + authorize_supplied_identity(&engine, &identity_cache, intent(), &identity("22")); + assert!( + matches!(replacement.action, NetworkAction::Deny { .. }), + "a new digest at an already trusted executable path must be denied" + ); + } + + #[test] + fn supplied_identity_rejects_replaced_authorizing_ancestor() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + allowed: + name: allowed + endpoints: + - host: api.example.com + port: 443 + binaries: + - path: /sandbox/bin/launcher +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .expect("load policy"); + let identity = |ancestor_digest: &str| { + Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: PathBuf::from("/sandbox/bin/client"), + digest: Some("11".repeat(32).parse().expect("digest")), + }, + ancestors: vec![ContractExecutableIdentity { + path: PathBuf::from("/sandbox/bin/launcher"), + digest: Some(ancestor_digest.repeat(32).parse().expect("digest")), + }], + cmdline_paths: Vec::new(), + }) + }; + let intent = || EgressIntent::connect("api.example.com".to_string(), 443); + let identity_cache = BinaryIdentityCache::new(); + + let original = + authorize_supplied_identity(&engine, &identity_cache, intent(), &identity("22")); + assert!(matches!(original.action, NetworkAction::Allow { .. })); + + let replacement = + authorize_supplied_identity(&engine, &identity_cache, intent(), &identity("33")); + assert!( + matches!(replacement.action, NetworkAction::Deny { .. }), + "a changed digest for an authorizing ancestor must be denied" + ); + } + + #[test] + fn supplied_identity_pin_survives_policy_reload() { + const POLICY_DATA: &str = r#" +network_policies: + allowed: + name: allowed + endpoints: + - host: api.example.com + port: 443 + binaries: + - path: /sandbox/bin/client +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#; + let rego = include_str!("../data/sandbox-policy.rego"); + let engine = OpaEngine::from_strings(rego, POLICY_DATA).expect("load policy"); + let identity = |digest_byte: &str| { + Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: PathBuf::from("/sandbox/bin/client"), + digest: Some(digest_byte.repeat(32).parse().expect("digest")), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }) + }; + let identity_cache = BinaryIdentityCache::new(); + let intent = || EgressIntent::connect("api.example.com".to_string(), 443); + + let original = + authorize_supplied_identity(&engine, &identity_cache, intent(), &identity("11")); + assert!(matches!(original.action, NetworkAction::Allow { .. })); + + engine.reload(rego, POLICY_DATA).expect("reload policy"); + let replacement = + authorize_supplied_identity(&engine, &identity_cache, intent(), &identity("22")); + + assert!(matches!(replacement.action, NetworkAction::Deny { .. })); + assert_eq!(replacement.policy_generation, 1); + assert!(replacement.endpoint.destination.is_none()); + assert!(replacement.endpoint.l7_route.is_none()); + assert!(replacement.endpoint.policy_configs.is_empty()); + assert!(replacement.endpoint.matched_endpoints.is_empty()); + } + + #[test] + fn supplied_identity_rejects_missing_ancestor_digest_before_policy() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + allowed: + name: allowed + endpoints: + - host: api.example.com + port: 443 + binaries: + - path: /sandbox/bin/client +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .expect("load policy"); + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: PathBuf::from("/sandbox/bin/client"), + digest: Some("11".repeat(32).parse().expect("digest")), + }, + ancestors: vec![ContractExecutableIdentity { + path: PathBuf::from("/sandbox/bin/launcher"), + digest: None, + }], + cmdline_paths: Vec::new(), + }); + + let decision = authorize_supplied_identity( + &engine, + &BinaryIdentityCache::new(), + EgressIntent::connect("api.example.com".to_string(), 443), + &identity, + ); + + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + assert!(decision.endpoint.destination.is_none()); + assert!(decision.endpoint.l7_route.is_none()); + assert!(decision.endpoint.policy_configs.is_empty()); + assert!(decision.endpoint.matched_endpoints.is_empty()); + } + #[tokio::test] async fn staged_transparent_open_waits_for_l4_policy() { let engine = OpaEngine::from_strings( @@ -6621,10 +6891,13 @@ process: "#, ) .unwrap(); + let identity_cache = BinaryIdentityCache::new(); let identity = || { Ok(ContractBinaryIdentity { - binary_path: PathBuf::from("/usr/bin/curl"), - binary_digest: Some("00".repeat(32).parse().unwrap()), + executable: ContractExecutableIdentity { + path: PathBuf::from("/usr/bin/curl"), + digest: Some("00".repeat(32).parse().unwrap()), + }, ancestors: Vec::new(), cmdline_paths: Vec::new(), }) @@ -6652,7 +6925,7 @@ process: let (allowed, allowed_result) = pending("203.0.113.7:443"); assert!( - preauthorize_transparent_open(allowed, None, &engine, None, None) + preauthorize_transparent_open(allowed, None, &engine, &identity_cache, None, None) .await .is_some() ); @@ -6660,9 +6933,16 @@ process: let (unsafe_destination, unsafe_result) = pending("169.254.169.254:80"); assert!( - preauthorize_transparent_open(unsafe_destination, None, &engine, None, None) - .await - .is_none() + preauthorize_transparent_open( + unsafe_destination, + None, + &engine, + &identity_cache, + None, + None, + ) + .await + .is_none() ); assert_eq!( unsafe_result.await.unwrap(), @@ -6671,7 +6951,7 @@ process: let (denied, denied_result) = pending("203.0.113.8:443"); assert!( - preauthorize_transparent_open(denied, None, &engine, None, None) + preauthorize_transparent_open(denied, None, &engine, &identity_cache, None, None) .await .is_none() ); @@ -6681,6 +6961,138 @@ process: ); } + #[tokio::test] + async fn staged_transparent_open_reports_invalid_identity_as_unavailable() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + allowed: + name: allowed + endpoints: + - host: 203.0.113.7 + port: 443 + binaries: + - path: /usr/bin/curl +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .unwrap(); + let identity_cache = BinaryIdentityCache::new(); + let (stream, _peer) = tokio::io::duplex(64); + let (decision, completion) = tokio::sync::oneshot::channel(); + let pending = PendingTcpOpen { + stream: Box::new(stream), + binary_identity: Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: PathBuf::from("/usr/bin/curl"), + digest: None, + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }), + destination: "203.0.113.7:443".parse().unwrap(), + socket: openshell_isolation_interface::contract::NetworkSocketMetadata { + socket_cookie: 7, + nonblocking: false, + process_generation: 1, + }, + policy_generation: engine.current_generation(), + timing: MediationTiming::default(), + decision, + }; + + assert!( + preauthorize_transparent_open(pending, None, &engine, &identity_cache, None, None) + .await + .is_none() + ); + assert_eq!( + completion.await.unwrap(), + TcpOpenDecision::Denied(TcpOpenDenial::IdentityUnavailable) + ); + } + + #[tokio::test] + async fn staged_transparent_open_reports_identity_cache_capacity_exhaustion() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + allowed: + name: allowed + endpoints: + - host: 203.0.113.7 + port: 443 + binaries: + - path: /sandbox/overflow +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .unwrap(); + let identity_cache = BinaryIdentityCache::new(); + for index in 0..4096 { + identity_cache + .verify_or_cache_supplied_identity(&ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: PathBuf::from(format!("/sandbox/pinned-{index}")), + digest: Some("11".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }) + .unwrap(); + } + let (stream, _peer) = tokio::io::duplex(64); + let (decision, completion) = tokio::sync::oneshot::channel(); + let pending = PendingTcpOpen { + stream: Box::new(stream), + binary_identity: Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: PathBuf::from("/sandbox/overflow"), + digest: Some("22".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }), + destination: "203.0.113.7:443".parse().unwrap(), + socket: openshell_isolation_interface::contract::NetworkSocketMetadata { + socket_cookie: 7, + nonblocking: false, + process_generation: 1, + }, + policy_generation: engine.current_generation(), + timing: MediationTiming::default(), + decision, + }; + + assert!( + preauthorize_transparent_open(pending, None, &engine, &identity_cache, None, None) + .await + .is_none() + ); + assert_eq!( + completion.await.unwrap(), + TcpOpenDecision::Denied(TcpOpenDenial::ResourceExhausted) + ); + } + struct FailedMediationSource; #[tokio::test] diff --git a/deploy/helm/openshell-workspace/templates/role.yaml b/deploy/helm/openshell-workspace/templates/role.yaml index bd0e33bad6..25d7b4f72e 100644 --- a/deploy/helm/openshell-workspace/templates/role.yaml +++ b/deploy/helm/openshell-workspace/templates/role.yaml @@ -43,3 +43,11 @@ rules: - pods verbs: - get + # Generation-scoped sandbox bootstrap Secrets. + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete diff --git a/deploy/helm/openshell-workspace/tests/workspace_test.yaml b/deploy/helm/openshell-workspace/tests/workspace_test.yaml index 7358e7de38..d4d3489f01 100644 --- a/deploy/helm/openshell-workspace/tests/workspace_test.yaml +++ b/deploy/helm/openshell-workspace/tests/workspace_test.yaml @@ -51,6 +51,18 @@ tests: resources: ["persistentvolumeclaims"] verbs: ["get"] + - it: grants generation Secret lifecycle and names no Secrets + template: templates/role.yaml + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] + - notExists: + path: rules[*].resourceNames + - it: binds the shared gateway service account template: templates/rolebinding.yaml set: diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index e2f7b3e6af..6f5439a575 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -209,6 +209,14 @@ discovery endpoint or its TLS CA. | certManager.serverIpAddresses | list | `["127.0.0.1"]` | IP SANs on the cert-manager-issued server certificate. | | certManager.serverIssuerRef | object | `{"group":"","kind":"","name":""}` | Override the issuerRef for the external server Certificate (e.g. a real LetsEncrypt/ACME ClusterIssuer for a publicly-trusted cert on an external hostname). When set, the chart creates a second server certificate from this issuer with only the hostnames in serverDnsNames; the internal server certificate is always signed by the chart's own CA. Leave name empty to use the chart CA for all server certificates (default). Requires certManager.enabled=true. | | fullnameOverride | string | `""` | Override the full generated resource name. | +| gateway.image.digest | string | `""` | Gateway image digest. When set, this takes precedence over tag. | +| gateway.image.pullPolicy | string | `nil` | Gateway image pull policy. Empty uses global.image.pullPolicy. | +| gateway.image.registry | string | `""` | Gateway image registry. Empty uses global.image.registry. | +| gateway.image.repository | string | `"openshell/gateway"` | Gateway image repository. | +| gateway.image.tag | string | `""` | Gateway image tag. Defaults to the chart appVersion when empty. | +| global.image.pullPolicy | string | `"IfNotPresent"` | Shared OpenShell image pull policy. Individual image pull policies take precedence. | +| global.image.registry | string | `"ghcr.io/nvidia"` | Shared OpenShell image registry. Individual image registries take precedence. | +| global.image.tag | string | `""` | Shared OpenShell image tag. Defaults to the chart appVersion when empty. | | grpcRoute.backendTLSPolicy.caCertificateConfigMapName | string | `""` | Name of the ConfigMap containing the CA certificate (key: ca.crt) used to validate the gateway pod's TLS certificate. Defaults to `-backend-ca` when empty. The certgen hook auto-creates this: with pkiInitJob (default), immediately on install/upgrade; with cert-manager, the hook polls for pkiInitJob.timeoutSeconds seconds waiting for cert-manager to issue the server certificate, then creates the ConfigMap. A single install usually succeeds; if cert-manager takes longer, increase pkiInitJob.timeoutSeconds. By default (pkiInitJob.failOnTimeout=true), the install fails if the timeout is reached; set failOnTimeout=false to allow the install to succeed and run `helm upgrade` after the certificate is issued. | | grpcRoute.backendTLSPolicy.enabled | bool | `false` | Create a BackendTLSPolicy resource for end-to-end TLS between the Gateway proxy and the OpenShell gateway pod. The traffic flow is: client → HTTPS → Gateway (terminate) → TLS (re-encrypt) → gateway pod. Requires server.disableTls=false and server.tls.enableMtls=false. The certgen hook auto-creates the backend CA ConfigMap. | | grpcRoute.backendTLSPolicy.hostname | string | `""` | Hostname the Gateway proxy validates against the backend's TLS certificate SAN. Defaults to the service FQDN (`..svc.cluster.local`) when empty, which matches the SAN included by both cert-manager and the pkiInitJob. | @@ -222,9 +230,6 @@ discovery endpoint or its TLS CA. | grpcRoute.gateway.name | string | `""` | Name of the Gateway resource. Defaults to the chart fullname. | | grpcRoute.gateway.namespace | string | `""` | Namespace of the Gateway referenced by the GRPCRoute parentRef. Defaults to the release namespace. | | grpcRoute.hostnames | list | `[]` | Hostnames the GRPCRoute matches on. Leave empty to match all hosts. | -| image.pullPolicy | string | `"IfNotPresent"` | Gateway image pull policy. | -| image.repository | string | `"ghcr.io/nvidia/openshell/gateway"` | Gateway image repository. | -| image.tag | string | `""` | Gateway image tag. Defaults to the chart appVersion when empty. | | imagePullSecrets | list | `[]` | Image pull secrets attached to gateway and helper pods. | | nameOverride | string | `"openshell"` | Override the chart name used in generated resource names. | | networkPolicy.enabled | bool | `true` | Restrict SSH ingress on sandbox pods to the gateway. In managed mode, the driver applies the equivalent policy to each workspace namespace. | @@ -254,9 +259,15 @@ discovery endpoint or its TLS CA. | probes.startup.timeoutSeconds | int | `1` | Startup probe timeout, in seconds. | | replicaCount | int | `1` | Number of OpenShell gateway replicas. Values greater than 1 require server.externalDbSecret because the default SQLite backend is per pod. | | resources | object | `{}` | Gateway pod resource requests and limits. | -| sandboxRuntime.image.pullPolicy | string | `""` | Sandbox runtime image pull policy. Defaults to the gateway image pull policy when empty. | -| sandboxRuntime.image.repository | string | `"ghcr.io/nvidia/openshell/sandbox"` | Sandbox runtime image repository. Changing it uses the effective gateway image tag unless tag is also set. | -| sandboxRuntime.image.tag | string | `""` | Sandbox runtime image tag override. Empty uses the version pinned into the gateway unless repository is changed. | +| sandbox.image.digest | string | `""` | Sandbox image digest. When set, this takes precedence over tag. | +| sandbox.image.pullPolicy | string | `nil` | Sandbox image pull policy. Leave unset to use the Kubernetes image default. | +| sandbox.image.repository | string | `"nvcr.io/nvidia/base/ubuntu"` | Default standalone sandbox image repository. | +| sandbox.image.tag | string | `"24.04"` | Sandbox image tag. Defaults to latest when empty. | +| sandboxRuntime.image.digest | string | `""` | Sandbox runtime image digest. When set, this takes precedence over tag. | +| sandboxRuntime.image.pullPolicy | string | `nil` | Sandbox runtime image pull policy. Empty uses global.image.pullPolicy. | +| sandboxRuntime.image.registry | string | `""` | Sandbox runtime image registry. Empty uses global.image.registry. | +| sandboxRuntime.image.repository | string | `"openshell/sandbox"` | Sandbox runtime image repository. | +| sandboxRuntime.image.tag | string | `""` | Sandbox runtime image tag. Defaults to the chart appVersion when empty. | | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | @@ -265,7 +276,7 @@ discovery endpoint or its TLS CA. | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | | securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. | | server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. | -| server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace | bool | `false` | Deprecated compatibility field. Credential storage no longer supports user-authored namespace references. | +| server.credentialDrivers.kubernetesSecrets.createNamespace | bool | `false` | Create the credential namespace. Requires a namespace other than the release namespace. The Namespace is retained on uninstall so stored credentials survive; an existing Namespace not owned by this release is left untouched. | | server.credentialDrivers.kubernetesSecrets.enabled | bool | `false` | Enable the in-tree Kubernetes Secret credential driver. WARNING: The RBAC Role grants read/write access to ALL Secrets in the configured namespace. Use a dedicated namespace to limit blast radius. | | server.credentialDrivers.kubernetesSecrets.namespace | string | `""` | Namespace where OpenShell-managed provider Secret objects are stored. Empty = Helm release namespace. A dedicated namespace is RECOMMENDED to isolate OpenShell-managed Secrets from other workloads. | | server.credentialDrivers.kubernetesSecrets.rbac.create | bool | `true` | Create a Role/RoleBinding granting the gateway ServiceAccount read/write access to managed provider Secrets. | @@ -314,8 +325,6 @@ discovery endpoint or its TLS CA. | server.policyValidationFailureMode | string | `"fail_closed"` | Posture when a candidate sandbox policy fails validation. `fail_closed` deactivates the previous policy; `retain_last_valid` keeps it active. | | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | -| server.sandboxImage | string | `"nvcr.io/nvidia/base/ubuntu:24.04"` | Default sandbox image used when requests do not specify one. | -| server.sandboxImagePullPolicy | string | `nil` | Pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | @@ -337,9 +346,11 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | -| supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | -| supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | +| supervisor.image.digest | string | `""` | Supervisor image digest. When set, this takes precedence over tag. | +| supervisor.image.pullPolicy | string | `nil` | Supervisor image pull policy. Empty uses global.image.pullPolicy. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | +| supervisor.image.registry | string | `""` | Supervisor image registry. Empty uses global.image.registry. | +| supervisor.image.repository | string | `"openshell/supervisor"` | Supervisor image repository. | +| supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | | supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. | | supervisor.sandboxRuntime.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. | | tolerations | list | `[]` | Tolerations for the gateway pod. | diff --git a/deploy/helm/openshell/ci/values-openshift-e2e.yaml b/deploy/helm/openshell/ci/values-openshift-e2e.yaml index d2aaf0fac8..c470f3ce84 100644 --- a/deploy/helm/openshell/ci/values-openshift-e2e.yaml +++ b/deploy/helm/openshell/ci/values-openshift-e2e.yaml @@ -28,8 +28,9 @@ # `allowUnauthenticatedUsers` only promotes the already cert-verified caller to a # dev principal at the app layer (mtls_auth is unsupported with the Kubernetes # driver). Both are required together; the client certificate is the access gate. -image: - pullPolicy: Always +gateway: + image: + pullPolicy: Always supervisor: image: diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 6f72265039..c3c1a84a4e 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -3,7 +3,6 @@ # Merge with values.yaml for Skaffold-driven local image builds (see skaffold.yaml). server: - sandboxImagePullPolicy: if_not_present otlp: endpoint: http://openshell-collector.observability.svc.cluster.local:4317 # Comment out to enforce mTLS (uses PKI secrets generated by pkiInitJob). @@ -11,6 +10,10 @@ server: auth: allowUnauthenticatedUsers: true +sandbox: + image: + pullPolicy: if_not_present + supervisor: image: pullPolicy: if_not_present diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index baa2e0f69e..2d7bf892e2 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -139,8 +139,11 @@ deploy: # To test multi-replica external PostgreSQL behavior: #- ci/values-high-availability.yaml setValueTemplates: - image.repository: '{{.IMAGE_REPO_openshell_gateway}}' - image.tag: '{{.IMAGE_TAG_openshell_gateway}}' + # Skaffold supplies complete image repositories. Clear the chart's + # shared registry so these values are used verbatim. + global.image.registry: '' + gateway.image.repository: '{{.IMAGE_REPO_openshell_gateway}}' + gateway.image.tag: '{{.IMAGE_TAG_openshell_gateway}}' supervisor.image.repository: '{{.IMAGE_REPO_openshell_supervisor}}' supervisor.image.tag: '{{.IMAGE_TAG_openshell_supervisor}}' sandboxRuntime.image.repository: '{{.IMAGE_REPO_openshell_sandbox}}' diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 67ed46d140..0f2a8d6149 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -43,7 +43,7 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 8 }} image: {{ include "openshell.image" . | quote }} - imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: {{ .Values.gateway.image.pullPolicy | default .Values.global.image.pullPolicy }} args: - --config - /etc/openshell/gateway.toml diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 03713b9c26..32e6b7cb63 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -86,12 +86,27 @@ default to enabled so upgrades with --reuse-values preserve the old topology. {{- if $enabled -}}true{{- end -}} {{- end }} -{{/* -Gateway image reference. Uses image.tag when set; falls back to .Chart.AppVersion -so a released chart automatically pulls the matching image without extra overrides. -*/}} +{{/* Gateway image reference. A digest takes precedence over a tag. */}} {{- define "openshell.image" -}} -{{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} +{{- $image := .Values.gateway.image -}} +{{- $global := .Values.global.image -}} +{{- $registry := $image.registry | default $global.registry -}} +{{- $repository := ternary (printf "%s/%s" $registry $image.repository) $image.repository (ne $registry "") -}} +{{- if $image.digest -}} +{{- printf "%s@%s" $repository $image.digest -}} +{{- else -}} +{{- printf "%s:%s" $repository ($image.tag | default $global.tag | default .Chart.AppVersion) -}} +{{- end }} +{{- end }} + +{{/* Sandbox image reference. A digest takes precedence over a tag. */}} +{{- define "openshell.sandboxImage" -}} +{{- $image := .Values.sandbox.image -}} +{{- if $image.digest -}} +{{- printf "%s@%s" $image.repository $image.digest -}} +{{- else -}} +{{- printf "%s:%s" $image.repository ($image.tag | default "latest") -}} +{{- end }} {{- end }} {{/* Official sandbox runtime repository used by the gateway's built-in default. */}} @@ -102,20 +117,23 @@ ghcr.io/nvidia/openshell/sandbox {{/* Whether Helm must propagate a sandbox runtime image override. */}} {{- define "openshell.sandboxRuntimeImageOverrideEnabled" -}} {{- $defaultRepository := include "openshell.defaultSandboxRuntimeRepository" . -}} -{{- $repository := .Values.sandboxRuntime.image.repository | default $defaultRepository -}} -{{- if or (ne $repository $defaultRepository) .Values.sandboxRuntime.image.tag -}}true{{- end -}} +{{- $global := .Values.global.image -}} +{{- $registry := .Values.sandboxRuntime.image.registry | default $global.registry -}} +{{- $repository := ternary (printf "%s/%s" $registry .Values.sandboxRuntime.image.repository) .Values.sandboxRuntime.image.repository (ne $registry "") -}} +{{- if or (ne $repository $defaultRepository) .Values.sandboxRuntime.image.tag .Values.sandboxRuntime.image.digest .Values.global.image.tag -}}true{{- end -}} {{- end }} {{/* Sandbox runtime image override. */}} {{- define "openshell.sandboxRuntimeImage" -}} -{{- $repository := .Values.sandboxRuntime.image.repository | default (include "openshell.defaultSandboxRuntimeRepository" .) -}} -{{- $tag := .Values.sandboxRuntime.image.tag | default .Values.image.tag | default .Chart.AppVersion -}} -{{- printf "%s:%s" $repository $tag }} -{{- end }} - -{{/* Official supervisor repository used by the gateway's built-in default. */}} -{{- define "openshell.defaultSupervisorRepository" -}} -ghcr.io/nvidia/openshell/supervisor +{{- $global := .Values.global.image -}} +{{- $registry := .Values.sandboxRuntime.image.registry | default $global.registry -}} +{{- $repository := ternary (printf "%s/%s" $registry .Values.sandboxRuntime.image.repository) .Values.sandboxRuntime.image.repository (ne $registry "") -}} +{{- if .Values.sandboxRuntime.image.digest -}} +{{- printf "%s@%s" $repository .Values.sandboxRuntime.image.digest -}} +{{- else -}} +{{- $tag := .Values.sandboxRuntime.image.tag | default $global.tag | default .Chart.AppVersion -}} +{{- printf "%s:%s" $repository $tag -}} +{{- end -}} {{- end }} {{/* @@ -133,24 +151,17 @@ true {{- end -}} {{- end -}} -{{/* -Whether Helm must propagate a supervisor image override into gateway.toml. -The chart's documented repository and empty tag are the gateway-owned default. -*/}} -{{- define "openshell.supervisorImageOverrideEnabled" -}} -{{- $defaultRepository := include "openshell.defaultSupervisorRepository" . -}} -{{- $repository := .Values.supervisor.image.repository | default $defaultRepository -}} -{{- if or (ne $repository $defaultRepository) .Values.supervisor.image.tag -}}true{{- end -}} -{{- end }} - -{{/* -Supervisor image override. A tag-only override uses the official repository; -a repository-only override uses the effective gateway image tag. -*/}} +{{/* Supervisor image override. */}} {{- define "openshell.supervisorImage" -}} -{{- $repository := .Values.supervisor.image.repository | default (include "openshell.defaultSupervisorRepository" .) -}} -{{- $tag := .Values.supervisor.image.tag | default .Values.image.tag | default .Chart.AppVersion -}} -{{- printf "%s:%s" $repository $tag }} +{{- $global := .Values.global.image -}} +{{- $registry := .Values.supervisor.image.registry | default $global.registry -}} +{{- $repository := ternary (printf "%s/%s" $registry .Values.supervisor.image.repository) .Values.supervisor.image.repository (ne $registry "") -}} +{{- if .Values.supervisor.image.digest -}} +{{- printf "%s@%s" $repository .Values.supervisor.image.digest -}} +{{- else -}} +{{- $tag := .Values.supervisor.image.tag | default $global.tag | default .Chart.AppVersion -}} +{{- printf "%s:%s" $repository $tag -}} +{{- end }} {{- end }} {{/* @@ -169,6 +180,27 @@ Namespace where sandbox pods are created. An explicit {{- .Values.server.sandboxNamespace | default .Release.Namespace -}} {{- end }} +{{/* +Secrets in the sandbox namespace whose contents the Kubernetes driver stages +into per-generation Secrets in workspace namespaces, as a JSON array. Empty in +shared workspace mode. +*/}} +{{- define "openshell.workspaceSecretSourceNames" -}} +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- $names := list -}} +{{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) -}} +{{- $names = append $names .Values.server.tls.clientTlsSecretName -}} +{{- end -}} +{{- if eq $workspaceMode "managed" -}} +{{- range .Values.server.sandboxImagePullSecrets -}} +{{- if .name -}} +{{- $names = append $names .name -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- uniq $names | toJson -}} +{{- end }} + {{/* Namespace where Kubernetes Secret-backed provider credentials live. */}} diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index b2ce8d2a82..f7c9a751d3 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -86,7 +86,7 @@ spec: containers: - name: certgen image: {{ include "openshell.image" . | quote }} - imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: {{ .Values.gateway.image.pullPolicy | default .Values.global.image.pullPolicy }} securityContext: allowPrivilegeEscalation: false capabilities: @@ -157,7 +157,7 @@ spec: containers: - name: certgen image: {{ include "openshell.image" . | quote }} - imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: {{ .Values.gateway.image.pullPolicy | default .Values.global.image.pullPolicy }} securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 8bd68eeb5c..17c483deb9 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -89,70 +89,22 @@ rules: - patch - watch {{- end }} - {{- $copiedSecretNames := list }} - {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }} - {{- $copiedSecretNames = append $copiedSecretNames .Values.server.tls.clientTlsSecretName }} - {{- end }} - {{- if eq $workspaceMode "managed" }} - {{- range .Values.server.sandboxImagePullSecrets }} - {{- if .name }} - {{- $copiedSecretNames = append $copiedSecretNames .name }} - {{- end }} - {{- end }} - {{- end }} {{- if ne $workspaceMode "shared" }} - apiGroups: [""] resources: ["services"] verbs: ["create", "get"] - - apiGroups: [""] - resources: ["secrets"] - # Bootstrap Secrets are generation-scoped. Recovery lists them by label and - # deletes stale generations with UID preconditions before retrying creation. - verbs: ["create", "list", "delete"] - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] {{- end }} - {{- $copiedSecretNames = uniq $copiedSecretNames }} - {{- if $copiedSecretNames }} - # Copy only explicitly configured TLS and image-pull Secrets into workspace - # namespaces. Server-side apply authorizes these requests as patch operations. - - apiGroups: - - "" - resources: - - secrets - resourceNames: - {{- range $copiedSecretNames }} - - {{ . | quote }} - {{- end }} - verbs: - - get - - patch - # Server-side apply uses PATCH for an existing Secret, but the API server - # additionally authorizes CREATE when the named Secret does not exist yet. - # Kubernetes RBAC cannot restrict CREATE by resourceNames because create - # authorization happens before the object name is available to RBAC. Keep - # reads and mutations name-restricted above; this broader grant is required - # only to create the initial copy in a gateway-owned managed namespace. - - apiGroups: - - "" - resources: - - secrets - verbs: - - create - {{- end }} - {{- if and (ne $workspaceMode "shared") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - # The kubernetes-secrets credential driver uses dynamic hashed names in - # workspace namespaces, so Kubernetes RBAC cannot restrict resourceNames. - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create - - patch - - delete + {{- if eq $workspaceMode "managed" }} + # Operator-mode namespaces receive Secret permissions from the openshell-workspace + # chart Role instead. + - apiGroups: [""] + resources: ["secrets"] + # Bootstrap and image-pull Secrets are generation-scoped. Recovery deletes + # them by exact name. + verbs: ["create", "delete"] {{- end }} {{- if eq $workspaceMode "managed" }} # ServiceAccount creation in managed namespaces. diff --git a/deploy/helm/openshell/templates/credential-secrets-namespace.yaml b/deploy/helm/openshell/templates/credential-secrets-namespace.yaml new file mode 100644 index 0000000000..e5a1a3cd2e --- /dev/null +++ b/deploy/helm/openshell/templates/credential-secrets-namespace.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- $credentialSecrets := .Values.server.credentialDrivers.kubernetesSecrets -}} +{{- if and $credentialSecrets.enabled $credentialSecrets.createNamespace }} +{{- $namespace := include "openshell.credentialKubernetesSecretsNamespace" . -}} +{{- if eq $namespace .Release.Namespace -}} +{{- fail "server.credentialDrivers.kubernetesSecrets.createNamespace requires server.credentialDrivers.kubernetesSecrets.namespace to name a namespace other than the release namespace" -}} +{{- end -}} +{{- $existing := lookup "v1" "Namespace" "" $namespace -}} +{{- $releaseName := dig "metadata" "annotations" "meta.helm.sh/release-name" "" $existing -}} +{{- $releaseNamespace := dig "metadata" "annotations" "meta.helm.sh/release-namespace" "" $existing -}} +{{- if or (not $existing) (and (eq $releaseName .Release.Name) (eq $releaseNamespace .Release.Namespace)) }} +# Retained on uninstall so removing the gateway cannot delete stored +# credentials. A retained Namespace keeps its Helm ownership annotations, so a +# reinstall of the same release adopts it; a Namespace owned by anything else +# is left alone. +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +{{- end }} +{{- end }} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 8c50b4053c..13b453041b 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -140,13 +140,11 @@ data: [openshell.drivers.kubernetes] allow_driver_config = {{ .Values.server.drivers.kubernetes.allowDriverConfig }} namespace = {{ include "openshell.sandboxNamespace" . | quote }} - default_image = {{ .Values.server.sandboxImage | quote }} + default_image = {{ include "openshell.sandboxImage" . | quote }} {{- if include "openshell.sandboxRuntimeImageOverrideEnabled" . }} sandbox_runtime_image = {{ include "openshell.sandboxRuntimeImage" . | quote }} {{- end }} - {{- if include "openshell.supervisorImageOverrideEnabled" . }} supervisor_image = {{ include "openshell.supervisorImage" . | quote }} - {{- end }} {{- if .Values.server.hostGatewayIP }} host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} {{- end }} @@ -191,8 +189,8 @@ data: {{- if .Values.server.providerTokenGrants.spiffe.enabled }} provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} {{- end }} - {{- if .Values.server.sandboxImagePullPolicy }} - image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.server.sandboxImagePullPolicy | quote }} + {{- if .Values.sandbox.image.pullPolicy }} + image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.sandbox.image.pullPolicy | quote }} {{- end }} {{- $sandboxImagePullSecretNames := list -}} {{- range .Values.server.sandboxImagePullSecrets }} @@ -212,11 +210,11 @@ data: {{- if .Values.server.defaultRuntimeClassName }} default_runtime_class_name = {{ .Values.server.defaultRuntimeClassName | quote }} {{- end }} - {{- if .Values.supervisor.image.pullPolicy }} - supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.supervisor.image.pullPolicy | quote }} + {{- if (.Values.supervisor.image.pullPolicy | default .Values.global.image.pullPolicy) }} + supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" (.Values.supervisor.image.pullPolicy | default .Values.global.image.pullPolicy) | quote }} {{- end }} - {{- if .Values.sandboxRuntime.image.pullPolicy }} - sandbox_runtime_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.sandboxRuntime.image.pullPolicy | quote }} + {{- if (.Values.sandboxRuntime.image.pullPolicy | default .Values.global.image.pullPolicy) }} + sandbox_runtime_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" (.Values.sandboxRuntime.image.pullPolicy | default .Values.global.image.pullPolicy) | quote }} {{- end }} [openshell.drivers.kubernetes.resource_admission] @@ -247,9 +245,6 @@ data: [openshell.credential_drivers.kubernetes-secrets] namespace = {{ include "openshell.credentialKubernetesSecretsNamespace" . | quote }} - allow_reference_namespace = {{ .Values.server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace }} - workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} - gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} {{- end }} {{- if .Values.server.credentialDrivers.vault.enabled }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index d43f5de0ab..b47efa090b 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -59,9 +59,9 @@ rules: verbs: ["create", "get"] - apiGroups: [""] resources: ["secrets"] - # Bootstrap Secrets are generation-scoped. Recovery lists them by label and - # deletes stale generations with UID preconditions before retrying creation. - verbs: ["create", "list", "delete"] + # Bootstrap Secrets are generation-scoped. Recovery deletes them by exact + # name. + verbs: ["create", "delete"] - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] diff --git a/deploy/helm/openshell/templates/workspace-secret-source-role.yaml b/deploy/helm/openshell/templates/workspace-secret-source-role.yaml new file mode 100644 index 0000000000..627fbbaa7e --- /dev/null +++ b/deploy/helm/openshell/templates/workspace-secret-source-role.yaml @@ -0,0 +1,25 @@ +{{- $sourceSecretNames := include "openshell.workspaceSecretSourceNames" . | fromJsonArray }} +{{- if $sourceSecretNames }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Read the TLS and image-pull Secrets the gateway stages into workspace +# namespaces. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "openshell.fullname" . }}-workspace-secret-source + namespace: {{ include "openshell.sandboxNamespace" . }} + labels: + {{- include "openshell.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + resourceNames: + {{- range $sourceSecretNames }} + - {{ . | quote }} + {{- end }} + verbs: + - get +{{- end }} diff --git a/deploy/helm/openshell/templates/workspace-secret-source-rolebinding.yaml b/deploy/helm/openshell/templates/workspace-secret-source-rolebinding.yaml new file mode 100644 index 0000000000..61dcfd2ada --- /dev/null +++ b/deploy/helm/openshell/templates/workspace-secret-source-rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if include "openshell.workspaceSecretSourceNames" . | fromJsonArray }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "openshell.fullname" . }}-workspace-secret-source + namespace: {{ include "openshell.sandboxNamespace" . }} + labels: + {{- include "openshell.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "openshell.fullname" . }}-workspace-secret-source +subjects: + - kind: ServiceAccount + name: {{ include "openshell.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 528efd7cba..326bc2cb60 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -79,50 +79,28 @@ tests: resources: ["networkpolicies"] verbs: ["get", "create", "patch", "update"] - - it: grants broad secret access when credential driver is enabled (operator) + - it: omits credential Secret access in managed mode set: - server.drivers.kubernetes.workspaceMode: operator + server.drivers.kubernetes.workspaceMode: managed server.credentialDrivers.kubernetesSecrets.enabled: true asserts: - - contains: + - notContains: path: rules content: apiGroups: [""] resources: ["secrets"] verbs: ["get", "create", "patch", "delete"] - - it: restricts operator TLS sync to the configured secret + - it: omits all Secret rules in operator mode set: server.drivers.kubernetes.workspaceMode: operator server.tls.clientTlsSecretName: custom-client-tls + server.credentialDrivers.kubernetesSecrets.enabled: true asserts: - - contains: - path: rules - content: - apiGroups: [""] - resources: ["secrets"] - resourceNames: ["custom-client-tls"] - verbs: ["get", "patch"] - - - it: grants operator bootstrap secret lifecycle permissions when TLS and credential storage are disabled - set: - server.drivers.kubernetes.workspaceMode: operator - server.disableTls: true - asserts: - - contains: - path: rules - content: - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "list", "delete"] - - notContains: - path: rules - content: - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create", "patch", "delete"] + - notExists: + path: rules[*].resources[?(@ == "secrets")] - - it: restricts managed copies to TLS and configured image-pull secrets + - it: names no Secrets in managed mode set: server.drivers.kubernetes.workspaceMode: managed server.tls.clientTlsSecretName: custom-client-tls @@ -130,51 +108,21 @@ tests: - name: registry-one - name: registry-two asserts: + - notExists: + path: rules[*].resourceNames - contains: path: rules content: apiGroups: [""] resources: ["secrets"] - resourceNames: ["custom-client-tls", "registry-one", "registry-two"] - verbs: ["get", "patch"] - - contains: - path: rules - content: - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "list", "delete"] - - - it: restricts managed copies to image-pull secrets when TLS is disabled - set: - server.drivers.kubernetes.workspaceMode: managed - server.disableTls: true - server.sandboxImagePullSecrets: - - name: registry-one - asserts: - - contains: - path: rules - content: - apiGroups: [""] - resources: ["secrets"] - resourceNames: ["registry-one"] - verbs: ["get", "patch"] - - contains: - path: rules - content: - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "list", "delete"] + verbs: ["create", "delete"] - it: omits secrets rule entirely in shared mode set: server.drivers.kubernetes.workspaceMode: shared asserts: - - notContains: - path: rules - content: - apiGroups: [""] - resources: ["secrets"] - any: true + - notExists: + path: rules[*].resources[?(@ == "secrets")] - it: grants managed sandbox-runtime companion permissions set: @@ -197,7 +145,7 @@ tests: content: apiGroups: [""] resources: ["secrets"] - verbs: ["create", "list", "delete"] + verbs: ["create", "delete"] - it: grants operator sandbox-runtime companion permissions set: @@ -215,9 +163,3 @@ tests: apiGroups: [""] resources: ["services"] verbs: ["create", "get"] - - contains: - path: rules - content: - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "list", "delete"] diff --git a/deploy/helm/openshell/tests/credential_drivers_test.yaml b/deploy/helm/openshell/tests/credential_drivers_test.yaml index 65829712d1..d7f1907c0d 100644 --- a/deploy/helm/openshell/tests/credential_drivers_test.yaml +++ b/deploy/helm/openshell/tests/credential_drivers_test.yaml @@ -66,7 +66,13 @@ tests: pattern: 'credential_drivers\s*=\s*\["kubernetes-secrets"\]' - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.credential_drivers\.kubernetes-secrets\].*?namespace\s*=\s*"provider-secrets".*?allow_reference_namespace\s*=\s*false' + pattern: '(?ms)\[openshell\.credential_drivers\.kubernetes-secrets\]\s*namespace\s*=\s*"provider-secrets"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'allow_reference_namespace' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '\[openshell\.credential_drivers\.kubernetes-secrets\][^\[]*(workspace_mode|gateway_id)' - notMatchRegex: path: data["gateway.toml"] pattern: 'transport\s*=\s*"in_tree"' diff --git a/deploy/helm/openshell/tests/credential_secrets_namespace_test.yaml b/deploy/helm/openshell/tests/credential_secrets_namespace_test.yaml new file mode 100644 index 0000000000..e102e35f72 --- /dev/null +++ b/deploy/helm/openshell/tests/credential_secrets_namespace_test.yaml @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: credential Secret namespace +templates: + - templates/credential-secrets-namespace.yaml +release: + name: openshell + namespace: my-namespace + +tests: + - it: does not create a namespace by default + set: + server.credentialDrivers.kubernetesSecrets.enabled: true + server.credentialDrivers.kubernetesSecrets.namespace: provider-secrets + asserts: + - hasDocuments: + count: 0 + + - it: creates a retained credential namespace + set: + server.credentialDrivers.kubernetesSecrets.enabled: true + server.credentialDrivers.kubernetesSecrets.namespace: provider-secrets + server.credentialDrivers.kubernetesSecrets.createNamespace: true + asserts: + - isKind: + of: Namespace + - equal: + path: metadata.name + value: provider-secrets + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + + - it: requires a namespace other than the release namespace + set: + server.credentialDrivers.kubernetesSecrets.enabled: true + server.credentialDrivers.kubernetesSecrets.createNamespace: true + asserts: + - failedTemplate: + errorPattern: createNamespace requires + + - it: leaves an existing namespace owned by something else untouched + set: + server.credentialDrivers.kubernetesSecrets.enabled: true + server.credentialDrivers.kubernetesSecrets.namespace: provider-secrets + server.credentialDrivers.kubernetesSecrets.createNamespace: true + kubernetesProvider: + scheme: + "v1/Namespace": + gvr: + version: "v1" + resource: "namespaces" + namespaced: false + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: provider-secrets + asserts: + - hasDocuments: + count: 0 + + - it: adopts a namespace retained from the same release + set: + server.credentialDrivers.kubernetesSecrets.enabled: true + server.credentialDrivers.kubernetesSecrets.namespace: provider-secrets + server.credentialDrivers.kubernetesSecrets.createNamespace: true + kubernetesProvider: + scheme: + "v1/Namespace": + gvr: + version: "v1" + resource: "namespaces" + namespaced: false + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: provider-secrets + annotations: + meta.helm.sh/release-name: openshell + meta.helm.sh/release-namespace: my-namespace + asserts: + - isKind: + of: Namespace diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 9ee2e26ee5..83fbb7aa08 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -82,6 +82,103 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?compute_driver\s*=\s*"kubernetes"' + - it: preserves a complete tagged gateway image reference + template: templates/statefulset.yaml + set: + gateway.image.registry: registry.example + gateway.image.repository: gateway + gateway.image.tag: branch + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/gateway:branch + + - it: defaults a repository-only gateway image to the chart appVersion + template: templates/statefulset.yaml + set: + gateway.image.registry: registry.example + gateway.image.repository: gateway + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/gateway:0.0.0 + + - it: applies global image values to gateway, supervisor, and sandbox runtime + template: templates/gateway-config.yaml + set: + global.image.registry: registry.example.com + global.image.tag: v0.1.0 + global.image.pullPolicy: IfNotPresent + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:v0\.1\.0"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_runtime_image\s*=\s*"registry\.example\.com/openshell/sandbox:v0\.1\.0"' + + - it: lets an individual registry override the global registry + template: templates/statefulset.yaml + set: + global.image.registry: registry.example.com + gateway.image.registry: mirror.example.com + gateway.image.repository: custom/gateway + global.image.tag: v0.1.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: mirror.example.com/custom/gateway:v0.1.0 + + - it: applies the global registry to the gateway image + template: templates/statefulset.yaml + set: + global.image.registry: registry.example.com + global.image.tag: v0.1.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/openshell/gateway:v0.1.0 + + - it: defaults a repository-only sandbox image to latest + template: templates/gateway-config.yaml + set: + sandbox.image.repository: registry.example/sandbox + sandbox.image.tag: "" + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'default_image\s*=\s*"registry\.example/sandbox:latest"' + + - it: gives a gateway digest precedence over its tag + template: templates/statefulset.yaml + set: + gateway.image.registry: registry.example + gateway.image.repository: gateway + gateway.image.tag: ignored + gateway.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/gateway@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + + - it: gives supervisor and sandbox digests precedence over their tags + template: templates/gateway-config.yaml + set: + supervisor.image.registry: registry.example + supervisor.image.repository: supervisor + supervisor.image.tag: ignored + supervisor.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + sandbox.image.repository: registry.example/sandbox + sandbox.image.tag: ignored + sandbox.image.digest: sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'registry\.example/supervisor@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - matchRegex: + path: data["gateway.toml"] + pattern: 'default_image\s*=\s*"registry\.example/sandbox@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"' + # Regression for Drew's P2: a ConfigMap-only mutation in `helm upgrade` # must roll the StatefulSet, otherwise pods keep running with stale config. - it: annotates the StatefulSet pod template with a ConfigMap checksum @@ -224,17 +321,17 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?grpc_endpoint' - - it: omits pull policies by default so Kubernetes applies its own defaults + - it: applies the global pull policy by default template: templates/gateway-config.yaml asserts: - - notMatchRegex: + - matchRegex: path: data["gateway.toml"] - pattern: '(?m)^\s*(image_pull_policy|supervisor_image_pull_policy)\s*=' + pattern: '(?ms)supervisor_image_pull_policy\s*=\s*"if_not_present".*?sandbox_runtime_image_pull_policy\s*=\s*"if_not_present"' - it: renders canonical image pull policies in the Kubernetes driver table template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: if_not_present + sandbox.image.pullPolicy: if_not_present supervisor.image.pullPolicy: never asserts: - matchRegex: @@ -244,7 +341,7 @@ tests: - it: translates legacy Kubernetes pull policy values to canonical gateway values template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: Always + sandbox.image.pullPolicy: Always supervisor.image.pullPolicy: IfNotPresent asserts: - matchRegex: @@ -254,7 +351,7 @@ tests: - it: rejects unsupported sandbox image pull policies template: templates/statefulset.yaml set: - server.sandboxImagePullPolicy: Sometimes + sandbox.image.pullPolicy: Sometimes asserts: - failedTemplate: errorMessage: 'image pull policy "Sometimes" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' @@ -344,12 +441,12 @@ tests: path: data["gateway.toml"] pattern: 'proxy_ca_bundle\s*=' - - it: uses the gateway built-in supervisor image by default + - it: renders the default supervisor image template: templates/gateway-config.yaml asserts: - - notMatchRegex: + - matchRegex: path: data["gateway.toml"] - pattern: 'supervisor_image\s*=' + pattern: 'supervisor_image\s*=\s*"ghcr\.io/nvidia/openshell/supervisor:0\.0\.0"' - it: uses the gateway built-in sandbox runtime image by default template: templates/gateway-config.yaml @@ -361,10 +458,12 @@ tests: - it: renders independent sandbox runtime and supervisor image overrides template: templates/gateway-config.yaml set: - sandboxRuntime.image.repository: registry.example.com/openshell/sandbox + sandboxRuntime.image.registry: registry.example.com + sandboxRuntime.image.repository: openshell/sandbox sandboxRuntime.image.tag: sandbox-build sandboxRuntime.image.pullPolicy: Always - supervisor.image.repository: registry.example.com/openshell/supervisor + supervisor.image.registry: registry.example.com + supervisor.image.repository: openshell/supervisor supervisor.image.tag: supervisor-build asserts: - matchRegex: @@ -377,6 +476,18 @@ tests: path: data["gateway.toml"] pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:supervisor-build"' + - it: gives a sandbox runtime digest precedence over its tag + template: templates/gateway-config.yaml + set: + sandboxRuntime.image.registry: registry.example.com + sandboxRuntime.image.repository: openshell/sandbox + sandboxRuntime.image.tag: ignored + sandboxRuntime.image.digest: sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_runtime_image\s*=\s*"registry\.example\.com/openshell/sandbox@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"' + - it: renders a supervisor tag override with the official repository template: templates/gateway-config.yaml set: @@ -386,20 +497,21 @@ tests: path: data["gateway.toml"] pattern: 'supervisor_image\s*=\s*"ghcr\.io/nvidia/openshell/supervisor:1\.2\.3"' - - it: renders a supervisor repository override with the effective gateway tag + - it: renders a supervisor repository override with the default chart tag template: templates/gateway-config.yaml set: - image.tag: gateway-build - supervisor.image.repository: registry.example.com/openshell/supervisor + supervisor.image.registry: registry.example.com + supervisor.image.repository: openshell/supervisor asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:gateway-build"' + pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:0\.0\.0"' - it: renders complete supervisor repository and tag overrides template: templates/gateway-config.yaml set: - supervisor.image.repository: registry.example.com/openshell/supervisor + supervisor.image.registry: registry.example.com + supervisor.image.repository: openshell/supervisor supervisor.image.tag: supervisor-build asserts: - matchRegex: @@ -1058,7 +1170,7 @@ tests: - it: accepts Always pull policy spelling for sandbox and supervisor template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: Always + sandbox.image.pullPolicy: Always supervisor.image.pullPolicy: Always asserts: - matchRegex: @@ -1068,7 +1180,7 @@ tests: - it: accepts IfNotPresent pull policy spelling for sandbox and supervisor template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: IfNotPresent + sandbox.image.pullPolicy: IfNotPresent supervisor.image.pullPolicy: IfNotPresent asserts: - matchRegex: @@ -1078,7 +1190,7 @@ tests: - it: accepts Never pull policy spelling for sandbox and supervisor template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: Never + sandbox.image.pullPolicy: Never supervisor.image.pullPolicy: Never asserts: - matchRegex: @@ -1088,7 +1200,7 @@ tests: - it: accepts canonical lowercase pull policies for sandbox and supervisor template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: always + sandbox.image.pullPolicy: always supervisor.image.pullPolicy: always asserts: - matchRegex: @@ -1098,7 +1210,7 @@ tests: - it: accepts canonical if_not_present pull policies for sandbox and supervisor template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: if_not_present + sandbox.image.pullPolicy: if_not_present supervisor.image.pullPolicy: if_not_present asserts: - matchRegex: @@ -1108,7 +1220,7 @@ tests: - it: accepts canonical never pull policies for sandbox and supervisor template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: never + sandbox.image.pullPolicy: never supervisor.image.pullPolicy: never asserts: - matchRegex: diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 36c2276e91..992d840922 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -73,7 +73,7 @@ tests: content: apiGroups: [""] resources: ["secrets"] - verbs: ["create", "list", "delete"] + verbs: ["create", "delete"] - notContains: path: rules content: diff --git a/deploy/helm/openshell/tests/workspace_secret_source_role_test.yaml b/deploy/helm/openshell/tests/workspace_secret_source_role_test.yaml new file mode 100644 index 0000000000..6eea72b992 --- /dev/null +++ b/deploy/helm/openshell/tests/workspace_secret_source_role_test.yaml @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: workspace Secret source RBAC +templates: + - templates/workspace-secret-source-role.yaml + - templates/workspace-secret-source-rolebinding.yaml +release: + name: openshell + namespace: my-namespace + +tests: + - it: renders nothing in shared mode + set: + server.drivers.kubernetes.workspaceMode: shared + server.sandboxImagePullSecrets: + - name: registry-one + asserts: + - hasDocuments: + count: 0 + + - it: grants reads of staged Secret sources in the sandbox namespace only + template: templates/workspace-secret-source-role.yaml + set: + server.drivers.kubernetes.workspaceMode: managed + server.sandboxNamespace: sandboxes + server.tls.clientTlsSecretName: custom-client-tls + server.sandboxImagePullSecrets: + - name: registry-one + asserts: + - isKind: + of: Role + - equal: + path: metadata.namespace + value: sandboxes + - equal: + path: rules + value: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["custom-client-tls", "registry-one"] + verbs: ["get"] + + - it: grants the operator-mode TLS Secret read without image-pull Secrets + template: templates/workspace-secret-source-role.yaml + set: + server.drivers.kubernetes.workspaceMode: operator + server.sandboxImagePullSecrets: + - name: registry-one + asserts: + - equal: + path: rules[0].resourceNames + value: ["openshell-client-tls"] + + - it: renders nothing in operator mode when TLS is disabled + set: + server.drivers.kubernetes.workspaceMode: operator + server.disableTls: true + asserts: + - hasDocuments: + count: 0 + + - it: binds the gateway ServiceAccount + template: templates/workspace-secret-source-rolebinding.yaml + set: + server.drivers.kubernetes.workspaceMode: managed + server.sandboxNamespace: sandboxes + asserts: + - equal: + path: metadata.namespace + value: sandboxes + - equal: + path: subjects[0].name + value: openshell + - equal: + path: subjects[0].namespace + value: my-namespace diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 84a59842a8..f8d1519bb7 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -7,6 +7,15 @@ # server.externalDbSecret because the default SQLite backend is per pod. replicaCount: 1 +global: + image: + # -- Shared OpenShell image registry. Individual image registries take precedence. + registry: ghcr.io/nvidia + # -- Shared OpenShell image tag. Defaults to the chart appVersion when empty. + tag: "" + # -- Shared OpenShell image pull policy. Individual image pull policies take precedence. + pullPolicy: IfNotPresent + workload: # -- Gateway workload controller kind. Use `statefulset` for the default # SQLite database, or `deployment` when server.externalDbSecret points at an @@ -18,41 +27,65 @@ workload: # StatefulSet identity or storage semantics. allowMultiReplicaStatefulSet: false -image: - # -- Gateway image repository. - repository: ghcr.io/nvidia/openshell/gateway - # -- Gateway image pull policy. - pullPolicy: IfNotPresent - # -- Gateway image tag. Defaults to the chart appVersion when empty. - tag: "" +gateway: + image: + # -- Gateway image registry. Empty uses global.image.registry. + registry: "" + # -- Gateway image repository. + repository: openshell/gateway + # -- Gateway image pull policy. Empty uses global.image.pullPolicy. + pullPolicy: null + # -- Gateway image tag. Defaults to the chart appVersion when empty. + tag: "" + # -- Gateway image digest. When set, this takes precedence over tag. + digest: "" # Trusted workload-side runtime image. sandboxRuntime: image: - # -- Sandbox runtime image repository. Changing it uses the effective gateway image tag unless tag is also set. - repository: ghcr.io/nvidia/openshell/sandbox - # -- Sandbox runtime image pull policy. Defaults to the gateway image pull policy when empty. - pullPolicy: "" - # -- Sandbox runtime image tag override. Empty uses the version pinned into the gateway unless repository is changed. + # -- Sandbox runtime image registry. Empty uses global.image.registry. + registry: "" + # -- Sandbox runtime image repository. + repository: openshell/sandbox + # -- Sandbox runtime image pull policy. Empty uses global.image.pullPolicy. + pullPolicy: null + # -- Sandbox runtime image tag. Defaults to the chart appVersion when empty. tag: "" + # -- Sandbox runtime image digest. When set, this takes precedence over tag. + digest: "" # Trusted control-side runtime image. supervisor: image: - # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. - repository: ghcr.io/nvidia/openshell/supervisor - # -- Sandbox supervisor pull policy. Leave unset to use the Kubernetes - # image default. Prefer always, if_not_present, or never; the chart also - # accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. + # -- Supervisor image registry. Empty uses global.image.registry. + registry: "" + # -- Supervisor image repository. + repository: openshell/supervisor + # -- Supervisor image pull policy. Empty uses global.image.pullPolicy. + # Prefer always, if_not_present, or never; the chart also accepts legacy + # Kubernetes spellings Always, IfNotPresent, and Never. pullPolicy: null - # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. + # -- Supervisor image tag. Defaults to the chart appVersion when empty. tag: "" + # -- Supervisor image digest. When set, this takes precedence over tag. + digest: "" sandboxRuntime: # -- Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. networkPolicyEnforced: false # -- Workload boundary TLS listener port. boundaryPort: 5500 +sandbox: + image: + # -- Default standalone sandbox image repository. + repository: nvcr.io/nvidia/base/ubuntu + # -- Sandbox image tag. Defaults to latest when empty. + tag: "24.04" + # -- Sandbox image digest. When set, this takes precedence over tag. + digest: "" + # -- Sandbox image pull policy. Leave unset to use the Kubernetes image default. + pullPolicy: null + # -- Operator-owned corporate forward proxy for policy-approved TLS egress # from Kubernetes sandboxes. The workload cannot select or override it. upstreamProxy: @@ -230,13 +263,6 @@ server: # from this Secret instead of using dbUrl. The Secret must contain a # `uri` key, e.g. postgresql://user:pass@host:5432/dbname. externalDbSecret: "" - # -- Default sandbox image used when requests do not specify one. - sandboxImage: "nvcr.io/nvidia/base/ubuntu:24.04" - # -- Pull policy for sandbox pods. Leave unset to use the Kubernetes image - # default (Always for :latest, IfNotPresent otherwise). Prefer always, - # if_not_present, or never; the chart also accepts legacy Kubernetes spellings - # Always, IfNotPresent, and Never. - sandboxImagePullPolicy: null # -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist # in the sandbox namespace. sandboxImagePullSecrets: [] @@ -342,8 +368,11 @@ server: # Empty = Helm release namespace. A dedicated namespace is RECOMMENDED # to isolate OpenShell-managed Secrets from other workloads. namespace: "" - # -- Deprecated compatibility field. Credential storage no longer supports user-authored namespace references. - allowReferenceNamespace: false + # -- Create the credential namespace. Requires a namespace other than the + # release namespace. The Namespace is retained on uninstall so stored + # credentials survive; an existing Namespace not owned by this release is + # left untouched. + createNamespace: false rbac: # -- Create a Role/RoleBinding granting the gateway ServiceAccount read/write access to managed provider Secrets. create: true diff --git a/deploy/kube/manifests/openshell-helmchart.yaml b/deploy/kube/manifests/openshell-helmchart.yaml index f2c7540333..8ab9d90597 100644 --- a/deploy/kube/manifests/openshell-helmchart.yaml +++ b/deploy/kube/manifests/openshell-helmchart.yaml @@ -24,14 +24,24 @@ spec: createNamespace: true valuesContent: |- chartChecksum: __CHART_CHECKSUM__ - image: - repository: ghcr.io/nvidia/openshell/gateway - tag: latest - pullPolicy: __IMAGE_PULL_POLICY__ + global: + image: + registry: ghcr.io/nvidia + gateway: + image: + repository: openshell/gateway + tag: latest + pullPolicy: __IMAGE_PULL_POLICY__ + supervisor: + image: + repository: openshell/supervisor + tag: latest + sandbox: + image: + repository: nvcr.io/nvidia/base/ubuntu + tag: "24.04" + pullPolicy: __SANDBOX_IMAGE_PULL_POLICY__ server: - sandboxImage: nvcr.io/nvidia/base/ubuntu:24.04 - sandboxImagePullPolicy: __SANDBOX_IMAGE_PULL_POLICY__ - supervisorImage: ghcr.io/nvidia/openshell/supervisor:latest dbUrl: __DB_URL__ hostGatewayIP: __HOST_GATEWAY_IP__ disableTls: __DISABLE_TLS__ diff --git a/docs/_components/CommandTerminal.tsx b/docs/_components/CommandTerminal.tsx index 9f8e75d0a0..27247b7be1 100644 --- a/docs/_components/CommandTerminal.tsx +++ b/docs/_components/CommandTerminal.tsx @@ -3,10 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -declare const React: unknown; - -const rotatingAgents = ["", "claude", "opencode", "codex"]; - export function CommandTerminal({ command }: { command: string }) { return (
-
$ - {"openshell sandbox create "} - - {rotatingAgents.map((agent, index) => ( - - {agent !== "" && ( - - {"-- "} - - {agent} - - - - )} - - ))} - + openshell sandbox create
diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index 9878fceb68..014af559ec 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -1,8 +1,8 @@ --- # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "How OpenShell Works" -sidebar-title: "How It Works" +title: "Architecture" +sidebar-title: "Architecture" description: "Understand the OpenShell architecture, runtime boundaries, gateways, sandboxes, and ecosystem integration points." keywords: "Generative AI, Cybersecurity, AI Agents, Architecture, Gateway, Sandbox, Providers" position: 2 @@ -90,7 +90,7 @@ device plugins without changing the gateway and sandbox contract. | [Gateways](/sandboxes/manage-gateways) | Authenticated control plane that owns API access, durable state, sandbox lifecycle, settings delivery, authorization, and relay coordination. | | [Providers](/sandboxes/manage-providers) | Credential and provider records that map logical agent needs to platform or user-managed secrets without exposing raw credentials to the agent process. | | [Policies](/sandboxes/policies) | Declarative controls for filesystem access, process identity, network egress, L7 rules, credential injection, and runtime policy updates. | -| [Provider-backed Inference](/sandboxes/inference-routing) | Per-sandbox provider attachment, native model endpoints, and endpoint-bound credential injection. | +| [Inference](/sandboxes/inference-routing) | Per-sandbox provider attachment, native model endpoints, and endpoint-bound credential injection. | ## Gateways and Sandboxes diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index e27d378381..9349ed51ca 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -3,11 +3,14 @@ # SPDX-License-Identifier: Apache-2.0 title: "Installation" sidebar-title: "Installation" -description: "Install OpenShell, choose a compute driver, and connect to a gateway." +description: "Install OpenShell, choose a sandbox runtime, and connect to a gateway." keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Installation, Setup, Gateway, Docker, Podman, MicroVM, Kubernetes" position: 3 --- +Install OpenShell on a local workstation, choose the runtime that runs +your sandboxes, and verify the package-managed gateway configuration. + ## Install OpenShell Install OpenShell with a single command: @@ -20,37 +23,35 @@ The script detects your operating system and installs the OpenShell CLI, standal You can also download release artifacts directly from the [OpenShell GitHub Releases](https://github.com/NVIDIA/OpenShell/releases) page. -### Install a prerelease - -Prerelease packages are retained as GitHub Actions artifacts for 90 days and require an authenticated [GitHub CLI](https://cli.github.com/) session. The `pre` alias installs the latest prerelease: - -```shell -gh auth login -curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | \ - OPENSHELL_VERSION=pre sh -``` - -The installer rejects expired candidates, downloads only the artifact required for your platform, and installs with Debian, RPM, or Homebrew. Installed packages retain the candidate's exact version, such as `0.1.0-pre.3`. Prerelease tags do not create entries on the GitHub Releases page. +Use `openshell status` to confirm the CLI can reach the gateway. -The `openshell` package on PyPI provides the Python SDK only. It does not install the `openshell` CLI. Add the SDK to a Python project with: +## Release Cadence -```shell -uv add openshell -``` +OpenShell publishes stable versions as coordinated release sets. The default +installer selects the newest stable release, and the `latest` documentation +channel follows that release. Use the same version of the gateway, compute and +credential drivers, supervisors, CLI, and SDK clients together. -Use `openshell status` to confirm the CLI can reach the gateway. +Between stable releases, numbered prereleases such as `0.1.0-pre.3` provide +evaluation checkpoints. Rolling development builds track successful releases +from `main` and use versions such as `0.0.0-dev.`. Prerelease and +development builds may change before the next stable release; their matching +documentation is published in the `dev` channel. -## Supported Compute Drivers +## Supported Runtimes -OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_driver` in the gateway TOML when you need to pin a specific driver. +OpenShell supports several sandbox runtimes. Package-managed gateways leave the +runtime unset by default so the gateway can auto-detect an available backend. +Set `compute_driver` in the gateway TOML when you need to pin a specific +runtime. -| Compute Driver | How It Is Configured | System Requirements | +| Runtime | How It Is Configured | System Requirements | |---|---|---| | Podman | The gateway is configured to create rootless Podman containers through the Podman API socket. | Linux with Podman 5.x, cgroups v2, rootless networking, and an active Podman user socket. | | Docker | The gateway is configured to create containers through Docker Desktop or Docker Engine. | Docker Desktop or Docker Engine 28.0 or later on the gateway host. | | MicroVM | The gateway is configured to create VM-backed sandboxes. | Host virtualization support. MicroVM uses Hypervisor.framework on macOS, KVM on Linux, and QEMU for GPU-backed sandboxes on Linux. | -For detailed driver behavior, refer to [Sandbox Compute Drivers](/reference/sandbox-compute-drivers). For gateway and sandbox operations, refer to [Gateways](/sandboxes/manage-gateways) and [Sandboxes](/sandboxes/manage-sandboxes). +For detailed runtime behavior, refer to [Sandbox Runtimes](/reference/sandbox-compute-drivers). For gateway and sandbox operations, refer to [Gateways](/sandboxes/manage-gateways) and [Sandboxes](/sandboxes/manage-sandboxes). ## macOS @@ -160,14 +161,6 @@ sudo systemctl restart snap.openshell.gateway Kubernetes deployments use the OpenShell Helm chart. For step-by-step installation, refer to [Kubernetes Setup](/kubernetes/setup). For chart values and packaging details, refer to the [Helm chart README](https://github.com/NVIDIA/OpenShell/blob/main/deploy/helm/openshell/README.md). -## Next Steps - -- To create your first sandbox, refer to the [Quickstart](/get-started/quickstart). -- To run the gateway as a container without the installer, refer to [Running the Gateway as a Container](/about/container-gateway). -- To register, select, and inspect gateways, refer to [Gateways](/sandboxes/manage-gateways). -- To supply API keys or tokens, refer to [Manage Providers](/sandboxes/manage-providers). -- To control what the agent can access, refer to [Policies](/sandboxes/policies). - ## Validate a package-managed gateway configuration Debian and Ubuntu packages validate the selected gateway configuration before @@ -194,3 +187,11 @@ Snap replays its effective daemon arguments through preflight and gives a nonemp `OPENSHELL_GATEWAY_CONFIG` precedence over `SNAP_COMMON/gateway.toml`. See [Gateway Configuration](/reference/gateway-config#gateway-config-preflight) for preflight details and manual schema-v1 migration steps. + +## Next Steps + +- To prepare an image and launch an agent, refer to [Run Your First Agent](/about/run-an-agent). +- To run the gateway as a container without the installer, refer to [Running the Gateway as a Container](/reference/container-gateway). +- To register, select, and inspect gateways, refer to [Gateways](/sandboxes/manage-gateways). +- To supply API keys or tokens, refer to [Manage Providers](/sandboxes/manage-providers). +- To control what the agent can access, refer to [Policies](/sandboxes/policies). diff --git a/docs/about/overview.mdx b/docs/about/overview.mdx index 4c8a234e73..3e8632080f 100644 --- a/docs/about/overview.mdx +++ b/docs/about/overview.mdx @@ -10,6 +10,12 @@ position: 1 NVIDIA OpenShell is an open-source runtime for executing autonomous AI agents in sandboxed environments with kernel-level isolation. It combines sandbox runtime controls and a declarative YAML policy so teams can run agents without giving them unrestricted access to local files, credentials, and external networks. + +New in OpenShell 0.1.0 is a [stable release cadence](/about/installation#release-cadence), a stronger [security model](/about/how-it-works), and an expanded [extension surface](/extensibility/extension-negotiation), along with much more. + +See our [upgrade guide](/upgrade/0-1-0) for everything that's changed. + + ## Why OpenShell Exists AI agents are most useful when they can read files, install packages, call APIs, and use credentials. That same access can create material risk. OpenShell is designed for this tradeoff: preserve agent capability while enforcing explicit controls over what the agent can access. @@ -54,5 +60,5 @@ OpenShell supports a range of agent deployment patterns. Explore these topics to go deeper: - To understand the runtime architecture, refer to [How OpenShell Works](/about/how-it-works). -- To install the CLI and create your first sandbox, refer to the [Quickstart](/get-started/quickstart). +- To prepare an image and launch an agent, refer to [Run Your First Agent](/about/run-an-agent). - To learn how OpenShell enforces policy controls across protection layers, refer to [Customize Sandbox Policies](/sandboxes/policies). diff --git a/docs/about/release-notes.mdx b/docs/about/release-notes.mdx deleted file mode 100644 index 62a3094b87..0000000000 --- a/docs/about/release-notes.mdx +++ /dev/null @@ -1,79 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "NVIDIA OpenShell Release Notes" -sidebar-title: "Release Notes" -description: "Track the latest changes and improvements to NVIDIA OpenShell." -keywords: "Generative AI, Cybersecurity, Release Notes, Changelog, AI Agents" -position: 6 ---- - -NVIDIA OpenShell follows a frequent release cadence. Use the following GitHub resources directly. - -## 0.1.0 migration notes - -### Workload images - -The default workload image is now `nvcr.io/nvidia/base/ubuntu:24.04`. It is a -minimal Ubuntu Noble environment and does not include agent CLIs or an -image-baked OpenShell policy. OpenShell applies its built-in restrictive policy -when no explicit policy is supplied. - -The CLI no longer expands bare catalog names or supports the former catalog -registry override. Build or select an OCI image and pass its explicit reference: - -```shell -openshell sandbox create --from registry.example.com/agents/my-agent:1.0 -- my-agent -``` - -Gateway operators can replace the default with the driver configuration or the -Helm `server.sandboxImage` value. - -### Network endpoint TLS handling - -OpenShell 0.1.0 closes the network endpoint `tls` field to two forms: omit it to auto-detect and terminate TLS for inspection, or set `skip` to stop inspecting that endpoint. The gateway, the sandbox supervisor, provider profile lint, and agent-authored policy proposals all reject any other value, including the former `terminate` and `passthrough` spellings. - -Remove the `tls` field from affected endpoints before upgrading. Removal preserves behavior, because `terminate` and `passthrough` already selected automatic termination. - -Do not substitute `tls: skip` for a removed `terminate`. `skip` stops TLS inspection, credential placeholder rewriting, dynamic token grant injection, and L7 rule enforcement for that endpoint, so it widens what the endpoint allows. - -A stored policy carrying a removed value fails validation on read, and affected sandboxes cannot start. Run `openshell policy list --global` to see which revision is invalid and why, then re-apply a corrected policy with `openshell policy set --global`. - -A policy baked into a workload image at `/etc/openshell/policy.yaml` is rejected the same way, because the supervisor requests it over the sandbox boundary when the gateway has no policy for the sandbox. Setting a policy with `openshell policy set --global` repairs that case too; rebuilding the image without the field restores the original behavior. - -### Provider profiles are import-only - -OpenShell 0.1.0 stops compiling provider profiles into the gateway binary. A gateway's catalog contains exactly the profiles an operator imported, the default `provider_profile_sources` is `[{ type = "user" }]`, and the `builtin` source type is removed: a `gateway.toml` that still names it is rejected at startup with the import command. A gateway with nothing imported reaches ready and serves an empty catalog. - -Export the profiles your deployment relies on **before** upgrading: - -```shell -openshell provider profile export -o yaml --global > .yaml -``` - -After upgrading, import them at the scope your providers use. You can also take them from the `providers/` directory of the matching release tag: - -```shell -openshell provider profile import --from ./providers --global -``` - -Omit `--global` for workspace-scoped providers. The canonical IDs are no longer reserved, so `github`, `openai`, `pypi` and the rest import at their own IDs, and the imported profile is the only definition for that ID. - -A provider whose profile is absent after the upgrade stays listable and exportable, but creating a sandbox with it, or attaching it to one, fails with a message naming the provider, the missing profile ID, and the import command. Legacy provider type aliases are gone with the compiled catalog: use a profile's own ID, so `--type claude-code` rather than `--type claude`, and `--type github` rather than `--type gh`. - -Command-to-provider inference is removed. `openshell sandbox create -- ` no longer derives a provider from the trailing command, and the compiled alias table that drove it is gone. A profile's `binaries` list authorizes a binary to reach that profile's endpoints; it is not a statement that running the binary asks for the provider, so treating it as attachment intent could attach credentials the operator did not select. Name providers explicitly with `--provider `, which still creates a missing provider from local discovery when the name matches an imported profile ID. - -The files under `providers/` are reviewable examples, not defaults. Read each file's header before importing it: several bind network access to binary paths from the image layout where they were authored, and imported unchanged into another image the profile matches nothing. - -### Network policy binaries - -OpenShell 0.1.0 removes the deprecated `NetworkBinary.harness` protobuf field and reserves its field number and name. Generated protobuf decoders ignore the unknown value. When the gateway loads policy history written by an older version, it migrates binaries marked by the former field into endpoint-provenance-marked rules so the upgrade cannot widen private-address access. - -Remove `harness` from sandbox policies and provider profiles before upgrading. Policy and profile YAML now reject the property. Provider profiles should list binaries as scalar paths, such as `- /usr/bin/curl`; the transitional object form `- path: /usr/bin/curl` remains accepted and is exported as a scalar. - -| Resource | Description | -|---|---| -| [Releases](https://github.com/NVIDIA/OpenShell/releases) | Versioned release notes and downloadable assets. | -| [Release comparison](https://github.com/NVIDIA/OpenShell/compare) | Diff between any two tags or branches. | -| [Merged pull requests](https://github.com/NVIDIA/OpenShell/pulls?q=is%3Apr+is%3Amerged) | Individual changes with review discussion. | -| [Commit history](https://github.com/NVIDIA/OpenShell/commits/main) | Full commit log on `main`. | diff --git a/docs/about/run-an-agent.mdx b/docs/about/run-an-agent.mdx new file mode 100644 index 0000000000..46581d54c5 --- /dev/null +++ b/docs/about/run-an-agent.mdx @@ -0,0 +1,132 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Run Your First Agent" +sidebar-title: "Run Your First Agent" +description: "Prepare a sandbox image, provider profile, and policy for an AI agent or another autonomous workload." +keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Provider Profiles, Policies, Custom Images" +position: 4 +--- + +OpenShell can run an AI agent or another autonomous command when its executable +is available in the sandbox image. The image, provider profile, and policy +define what the workload can execute and access. OpenShell does not require the +agent to use a specific framework or model API. + +## Prepare the Workload + +An agent needs three pieces: + +| Requirement | Purpose | +|---|---| +| Sandbox image | Contains the agent executable and its runtime dependencies. | +| Provider profile | Declares credential fields, service endpoints, and the executable paths allowed to use them. | +| Sandbox policy | Controls filesystem access, process behavior, and network destinations beyond access contributed by attached providers. | + +Build and maintain an OCI image for each workload. Install the agent, shell, +development tools, CA certificates, and language runtimes that the workload +needs. Use pinned versions so you can review and reproduce image updates. + +```shell +docker build -t registry.example.com/team/agent:1.0 . +docker push registry.example.com/team/agent:1.0 +openshell sandbox create --from registry.example.com/team/agent:1.0 +``` + +The `--from` option also accepts a local rootfs archive. It does not build a +Dockerfile or directory. Refer to +[Custom Containers](/sandboxes/manage-sandboxes#custom-containers) for image and +runtime details. The fallback image contains no agent or development toolchain, +so pass your image explicitly for agent workloads. + +## Prepare Provider Access + +A provider profile defines the credentials and endpoints an agent uses. The +gateway serves only profiles that an administrator or user has imported. +Review a profile before importing it, especially its executable paths and +network endpoints. + +```shell +curl -LsSfO https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/claude-code.yaml +openshell profile import -f claude-code.yaml --global +ANTHROPIC_API_KEY= \ + openshell provider create --name my-claude --type claude-code --from-existing +``` + +The repository includes example profiles in +[`providers/`](https://github.com/NVIDIA/OpenShell/tree/main/providers). Copy +and adapt an example when the agent binary is installed at another path or uses +a different service endpoint. Refer to [Profiles](/providers/profiles) for the +profile schema and import workflow. + +## Launch the Agent + +Pass the agent command after `--` and attach the provider instances it needs. +For example, this command starts Claude Code from an image your team built with +the `claude` executable installed: + +```shell +openshell sandbox create \ + --from registry.example.com/team/claude-code:1.0 \ + --provider my-claude \ + -- claude +``` + +The command before `--` configures the sandbox. The command after `--` becomes +the sandbox's main process. OpenShell keeps the sandbox after that process exits +unless you pass `--no-keep`. + +Use `--detach` for an unattended or long-running agent: + +```shell +openshell sandbox create \ + --name worker \ + --detach \ + --from my-registry.example.com/team/agent:latest \ + --provider model-provider \ + -- ./worker +``` + +Use a profile-backed provider for credentials that the agent should not read +directly. Plain values passed with `--env` are visible to the agent process. + +## Build an Agent Image + +Prepare each agent as part of your normal container build and release process: + +1. Select a trusted base image and install the agent executable and tools. +2. Create or adapt provider profiles for every external service the agent uses. +3. Add policy rules for required files, child processes, package registries, + tool servers, and other network destinations. +4. Build, scan, sign, and publish the image to a registry the gateway can pull. +5. Launch the executable as the sandbox's main process. + +Provider profiles can contribute endpoint and executable rules to the effective +policy. They do not grant unrelated network or filesystem access. Use +[Customize Sandbox Policies](/sandboxes/policies) when the agent needs access +beyond its attached providers. + +## Verify and Troubleshoot + +Inspect the sandbox, effective policy, provider attachments, and logs: + +```shell +openshell sandbox list +openshell policy get my-sandbox --full +openshell sandbox provider list my-sandbox +openshell logs my-sandbox --tail +``` + +If the executable is missing, change the sandbox image. If a provider request +is denied, confirm that the profile names the actual endpoint and executable +path. For another denied network or filesystem operation, update the sandbox +policy after reviewing the requested access. + +## Next Steps + +- For sandbox lifecycle, resources, templates, files, and connectivity, refer + to [Sandboxes](/sandboxes/manage-sandboxes). +- For the restrictive fallback policy, refer to + [Default Policy](/reference/default-policy). +- For operating-system and runtime requirements, refer to the + [Support Matrix](/reference/support-matrix). diff --git a/docs/about/supported-agents.mdx b/docs/about/supported-agents.mdx deleted file mode 100644 index 601a91ebe8..0000000000 --- a/docs/about/supported-agents.mdx +++ /dev/null @@ -1,22 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Supported Agents" -description: "AI agent frameworks and runtimes compatible with OpenShell sandboxes." -keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Claude, Codex, Cursor" -position: 5 ---- - -OpenShell can run containerized agents that support Linux. The default Ubuntu workload image does not bundle agent CLIs. Build or select an OCI image containing the agent, pass its explicit image reference to `--from`, and attach the providers and policy it needs. - -| Agent | Image workflow | Provider and policy requirements | -|---|---|---| -| [Claude Code](https://docs.anthropic.com/en/docs/claude-code) | Add Claude Code to your workload image. | Attach a `claude-code` provider or an endpoint-bearing profile for the selected model platform. Authorize the agent binary and service endpoints. | -| [OpenCode](https://opencode.ai/) | Add OpenCode to your workload image. | Attach the selected model provider and authorize OpenCode's binary paths and service endpoints. | -| [Codex](https://developers.openai.com/codex) | Add Codex to your workload image. | Attach an OpenAI provider and authorize the Codex binary paths and OpenAI endpoints. | -| [GitHub Copilot CLI](https://docs.github.com/en/copilot/github-copilot-in-the-cli) | Add GitHub Copilot CLI to your workload image. | Attach GitHub credentials and authorize the CLI binary and required GitHub endpoints. | -| [OpenClaw](https://openclaw.ai/) | Use the [NemoClaw](https://github.com/NVIDIA/NemoClaw) blueprint. | The blueprint manages the workload and its policy. | -| [Hermes Agent](https://github.com/NousResearch/hermes-agent) | Use the [NemoClaw](https://github.com/NVIDIA/NemoClaw) blueprint. | The blueprint manages the workload and its policy. | -| [Ollama](https://ollama.com/) | Build an image containing Ollama, or connect a sandbox to a separately managed Ollama service. | Authorize the client binary and the Ollama endpoint. Refer to [Inference with Ollama](/get-started/tutorials/inference-ollama). | - -For image selection and `--from` usage, refer to [Manage Sandboxes](/sandboxes/manage-sandboxes#sandbox-images). For a complete platform matrix, refer to [Support Matrix](/reference/support-matrix). diff --git a/docs/extensibility/drivers.mdx b/docs/extensibility/drivers.mdx new file mode 100644 index 0000000000..c988b651a6 --- /dev/null +++ b/docs/extensibility/drivers.mdx @@ -0,0 +1,40 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Drivers" +description: "Understand the compute and credential driver extension points in OpenShell." +keywords: "OpenShell Extensions, Compute Drivers, Credential Drivers, Extensibility" +--- + +OpenShell uses drivers to connect the gateway to workload runtimes and +credential stores. Drivers implement a stable capability boundary while the +gateway owns the public API, authorization, workspace scoping, and resource +lifecycle. + +## Compute Drivers + +Compute drivers create, inspect, and delete sandbox workloads. OpenShell ships +drivers for Kubernetes, Docker, Podman, virtual machines, and Windows MXC. An +external compute driver communicates with the gateway over a Unix domain +socket and negotiates its protocol version and capabilities before serving +requests. + +For built-in driver configuration and behavior, refer to +[Runtimes](/reference/sandbox-compute-drivers). + +## Credential Drivers + +Credential drivers store provider credentials behind opaque handles. The +gateway resolves those handles when a provider is attached without persisting +the secret value in the provider record. OpenShell includes database, +Kubernetes Secret, and Vault-compatible credential drivers. + +Configure the active credential driver in the +[Gateway Configuration](/reference/gateway-config#credential-drivers). + +## Compatibility + +Drivers exchange peer metadata with the gateway and advertise their extension +family capability. Upgrade both peers together when the protocol version +changes. Refer to [Extensibility Overview](/extensibility/extension-negotiation) +for the negotiation contract. diff --git a/docs/extensibility/extension-negotiation.mdx b/docs/extensibility/extension-negotiation.mdx index c8e3a1d791..eca4d6a47e 100644 --- a/docs/extensibility/extension-negotiation.mdx +++ b/docs/extensibility/extension-negotiation.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Extension Protocol Negotiation" -sidebar-title: "Protocol Negotiation" +sidebar-title: "Overview" description: "Implement version and capability negotiation for OpenShell extensions." keywords: "OpenShell Extensions, Protocol Version, Capabilities, Version Skew, Migration" --- diff --git a/docs/extensibility/isolation-backends.mdx b/docs/extensibility/isolation-backends.mdx new file mode 100644 index 0000000000..5b6856c956 --- /dev/null +++ b/docs/extensibility/isolation-backends.mdx @@ -0,0 +1,26 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Isolation Backends" +description: "Understand the runtime boundary used to launch and supervise sandboxed workloads." +keywords: "OpenShell Extensions, Isolation Backends, Sandbox Runtime, Extensibility" +--- + +An isolation backend connects the OpenShell supervisor to the runtime that +launches and controls a sandboxed workload. It defines how the supervisor +starts processes, attaches terminal streams, forwards signals, reports exit +status, and applies runtime-specific isolation. + +OpenShell separates this interface from compute drivers. A compute driver +provisions the workload environment, while the isolation backend controls +process execution inside that environment. This boundary lets runtimes evolve +without changing the gateway API or policy model. + +The OpenShell runtime backend implements the authenticated OpenShell Sandbox +Protocol used by the supervisor and `openshell-sandbox`. Backend selection and +capabilities remain internal to the workload runtime; users create and manage +sandboxes through the same gateway API. + +For the deployment-level runtime architecture, refer to +[Sandbox Runtime](/kubernetes/sandbox-runtime). For driver-specific workload +behavior, refer to [Runtimes](/reference/sandbox-compute-drivers). diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx deleted file mode 100644 index 55b1eb3ab4..0000000000 --- a/docs/get-started/quickstart.mdx +++ /dev/null @@ -1,100 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Quickstart" -description: "Install the OpenShell CLI, connect to a gateway, and create your first sandboxed AI agent." -keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Installation, Quickstart, Gateway, Docker, Kubernetes, Podman" -position: 1 ---- - -This page gets you from a reachable OpenShell gateway to a running, policy-enforced sandbox. - -## Prerequisites - -Before you begin, make sure you have: - -- A reachable OpenShell gateway. -- At least one compute driver configured for the gateway: Kubernetes, Docker, Podman, or MicroVM. -- The OpenShell CLI installed on your workstation. - -For a complete list of requirements, refer to [Support Matrix](/reference/support-matrix). -If you have not chosen a compute driver yet, refer to [Installation](/about/installation). - -## Install the OpenShell CLI - -Run the install script: - -```shell -curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh -``` - -The install script uses Homebrew, RPM, or a Debian package based on your machine. It starts the local gateway server after installation. - -After installing the CLI, run `openshell --help` in your terminal to view the full CLI reference. - - -Install the public OpenShell agent skills with `npx skills add NVIDIA/OpenShell`. The `openshell-cli` skill guides your agent through common workflows and uses the installed CLI help as the command reference; no OpenShell source checkout is required. - - -## Import a Provider Profile - -A provider profile describes the credentials, endpoints, and client binaries a -provider needs. A gateway serves only the profiles you imported, so import one -before creating a sandbox that uses a provider. - -The OpenShell repository ships example profiles in -[`providers/`](https://github.com/NVIDIA/OpenShell/tree/main/providers). Download -the one matching your agent and import it at platform scope: - -```shell -curl -LsSfO https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/claude-code.yaml -openshell provider profile import -f claude-code.yaml --global -``` - -Use `codex.yaml` for Codex, or `openai.yaml` for OpenCode. Confirm the catalog: - -```shell -openshell provider list-profiles -``` - - -Read the comment header at the top of each file before importing it. It names -the client binaries the profile expects and the image layout those paths assume. -If your sandbox image installs the agent somewhere else, edit `binaries` first — -otherwise the profile matches nothing and the credential is never injected. - - -## Create Your First Sandbox - -Create a sandbox with the gateway's default workload image: - -```shell -openshell sandbox create --name quickstart -``` - -The default is `nvcr.io/nvidia/base/ubuntu:24.04`. It provides a minimal Ubuntu Noble environment; install workload-specific tools in your own image. - -Connect to the sandbox: - -```shell -openshell sandbox connect quickstart -``` - -Run a command without opening an interactive session: - -```shell -openshell sandbox exec -n quickstart -- cat /etc/os-release -``` - -## Run an Agent Image - -Build an OCI image containing your agent and its dependencies, then pass its full image reference: - -```shell -docker build -t my-agent:latest ./my-agent -openshell sandbox create --from my-agent:latest -- my-agent -``` - -For a Podman gateway, build and use a Podman-visible name such as `localhost/my-agent:latest`. For a remote gateway, push the image to a registry that the gateway can pull from. - -Attach a provider and a policy that authorize the agent's credentials, endpoints, and executable paths. Refer to [Bring Your Own Container](https://github.com/NVIDIA/OpenShell/tree/main/examples/bring-your-own-container), [Providers](/providers), and [Customize Sandbox Policies](/sandboxes/policies). diff --git a/docs/get-started/tutorials/docker-compose.mdx b/docs/get-started/tutorials/docker-compose.mdx deleted file mode 100644 index 33f6b0c1d3..0000000000 --- a/docs/get-started/tutorials/docker-compose.mdx +++ /dev/null @@ -1,207 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Run the Gateway with Docker Compose" -sidebar-title: "Docker Compose Setup" -slug: "get-started/tutorials/docker-compose" -description: "Run the OpenShell gateway as a Docker Compose service and create agent sandboxes." -keywords: "Generative AI, Docker Compose, Gateway, Sandbox, OpenClaw, Docker, Installation" ---- - -This tutorial shows how to run the OpenShell gateway as a Docker Compose service on a Linux host or on a machine running Docker Desktop (Windows or macOS). - -After completing this tutorial you have: - -- An OpenShell gateway running as a Compose service. -- The `openshell` CLI registered against that gateway. -- An AI provider configured with your API key. -- A running agent sandbox. - -## Prerequisites - -- Docker Desktop (Windows or macOS) or Docker Engine with the Compose plugin (Linux). -- The `openshell` CLI installed on your workstation. See [Install the CLI](#install-the-cli) below. -- Port 8080 available on the host. - -## Compose files - -The Compose configuration lives at [`deploy/docker/`](https://github.com/NVIDIA/OpenShell/tree/main/deploy/docker) in the repository. - -| File | Purpose | -|---|---| -| `docker-compose.yml` | Gateway service, volumes, and environment variables | -| `gateway.toml` | TOML reference for release builds with config-file support | - -## Port note - -The Docker supervisor uses host networking and connects to the gateway through -the host's loopback port. The gateway listens on port 8080 inside the container, -so **port 8080 must be published at the same number on the Docker host**. -Publishing it as a different host port (for example `18080:8080`) leaves the -supervisor unable to establish its gateway session and the sandbox remains in -the `Provisioning` phase. - -If port 8080 is taken, change `OPENSHELL_SERVER_PORT` and update the port mapping to `:8080`, then set `OPENSHELL_PORT=` in an `.env` file. - -## Data directory - -The gateway extracts the `openshell-sandbox` supervisor binary from `ghcr.io/nvidia/openshell/supervisor:latest` on first start and caches it at: - -```text -/var/lib/openshell/openshell/docker-supervisor//openshell-sandbox -``` - -This path is used as a bind-mount source when Docker creates sandbox containers. -Docker resolves bind-mount sources against the **host filesystem**, not the container filesystem, so the data directory must be bind-mounted at the **same absolute path** in both the host and the container. - -The Compose file uses `/var/lib/openshell` for this purpose and sets `create_host_path: true` so Docker creates it on first run. - -## Start the gateway - -```shell -cd deploy/docker -docker compose up -d -``` - -Verify the gateway is healthy: - -```shell -curl -sf http://localhost:8080/healthz -``` - -## Install the CLI - -**Binary (recommended — macOS / Linux / WSL):** - -```shell -curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh -``` - - -The `openshell` package on PyPI provides the Python SDK only and does not install the CLI. - -On Windows without WSL, install the CLI inside a WSL 2 distribution (for example AlmaLinux or Ubuntu) and run all `openshell` commands from that distribution. - - -## Register the gateway - -Run this once after the gateway starts: - -```shell -openshell gateway add http://localhost:8080 --name openshell-docker -``` - -Verify the connection: - -```shell -openshell status -``` - -The output should show `Status: Connected`. - -## Configure an AI provider - -Import the provider profile for the service you use, then set your API key as an -environment variable and create a provider. A gateway serves only the profiles -you imported: - -```shell -curl -LsSfO https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/anthropic.yaml -openshell provider profile import -f anthropic.yaml --global -``` - -Use `openai.yaml` for the OpenAI tab below. - - - - -```shell -ANTHROPIC_API_KEY=sk-ant-... \ - openshell provider create --name anthropic --type anthropic --from-existing -``` - - - - -```shell -OPENAI_API_KEY=sk-... \ - openshell provider create --name openai --type openai --from-existing -``` - - - - -Confirm the provider was stored: - -```shell -openshell provider list -``` - -## Pre-pull workload images (optional) - -Workload images are pulled automatically on first use. Pre-pull the default -Ubuntu workload or your own agent image to avoid waiting during sandbox -creation: - -```shell -docker pull nvcr.io/nvidia/base/ubuntu:24.04 -docker pull registry.example.com/your-org/agent:latest -``` - -## Create a Sandbox - -Create a minimal Ubuntu sandbox: - -```shell -openshell sandbox create --name compose-demo -``` - -To run an agent, use an explicit OCI image that contains it: - -```shell -openshell sandbox create \ - --from registry.example.com/your-org/agent:latest \ - --provider \ - -- my-agent -``` - -OpenClaw runs through [NemoClaw](https://github.com/NVIDIA/NemoClaw), which -manages its sandbox image, model provider, and security policies. Follow the -[NemoClaw Quickstart](https://docs.nvidia.com/nemoclaw/latest/get-started/quickstart/) -for that workflow. - -Wait for the phase to change from `Provisioning` to `Ready`: - -```shell -openshell sandbox list -``` - -Then connect: - -```shell -openshell sandbox connect -``` - -## Manage the gateway - -| Command | Purpose | -|---|---| -| `docker compose up -d` | Start or restart the gateway | -| `docker compose down` | Stop the gateway and remove the container | -| `docker compose logs -f` | Tail gateway logs | -| `docker compose pull` | Pull a new gateway image version | - -## Linux notes - -On Linux, `host.docker.internal` and `host.openshell.internal` are not automatically resolvable from containers. Add the following under the `gateway` service in `docker-compose.yml`: - -```yaml -extra_hosts: - - "host.docker.internal:host-gateway" - - "host.openshell.internal:host-gateway" -``` - -## Next steps - -- [First Network Policy](/get-started/tutorials/first-network-policy) — apply L7 policies to your sandbox. -- [GitHub Push Access](/get-started/tutorials/github-sandbox) — grant a sandbox scoped GitHub access. diff --git a/docs/get-started/tutorials/first-network-policy.mdx b/docs/get-started/tutorials/first-network-policy.mdx index cbf9eaaf92..1a5cf67786 100644 --- a/docs/get-started/tutorials/first-network-policy.mdx +++ b/docs/get-started/tutorials/first-network-policy.mdx @@ -19,8 +19,9 @@ After completing this tutorial, you understand: ## Prerequisites -- A working OpenShell installation. Complete the [Quickstart](/get-started/quickstart) before proceeding. +- A working OpenShell installation with an active gateway. Refer to [Installation](/about/installation) before proceeding. - Docker Desktop running on your machine. +- A user-owned workload image with `curl` installed. To run every step of this tutorial, you can also use the automated demo script at the [examples/sandbox-policy-quickstart](https://github.com/NVIDIA/OpenShell/blob/main/examples/sandbox-policy-quickstart) directory in the NVIDIA OpenShell repository. It runs the full walkthrough in under a minute but without any user interaction. @@ -38,7 +39,10 @@ bash examples/sandbox-policy-quickstart/demo.sh Start by creating a sandbox with no network policies. This gives you a clean environment to observe default-deny behavior. ```shell -openshell sandbox create --name demo --no-auto-providers +openshell sandbox create \ + --name demo \ + --from registry.example.com/team/agent-tools:1.0 \ + --no-auto-providers ``` `--no-auto-providers` skips the provider setup prompt since this tutorial uses `curl` instead of an AI agent. diff --git a/docs/get-started/tutorials/github-sandbox.mdx b/docs/get-started/tutorials/github-sandbox.mdx index 6be5b3445c..2e52826e77 100644 --- a/docs/get-started/tutorials/github-sandbox.mdx +++ b/docs/get-started/tutorials/github-sandbox.mdx @@ -4,226 +4,140 @@ title: "Grant GitHub Push Access to a Sandboxed Agent" sidebar-title: "GitHub Push Access" slug: "get-started/tutorials/github-sandbox" -description: "Learn the iterative policy workflow by launching a sandbox, diagnosing a GitHub access denial, and applying a custom policy to fix it." +description: "Launch an agent with a GitHub provider, diagnose a denied push, and grant repository-scoped write access." keywords: "Generative AI, Cybersecurity, Tutorial, GitHub, Sandbox, Policy, Claude Code" --- -This tutorial walks through an iterative sandbox policy workflow. You launch a sandbox, ask Claude Code to push code to GitHub, and observe the default network policy denying the request. -You then diagnose the denial from your machine and from inside the sandbox, apply a policy update, and verify that the policy update to the sandbox takes effect. +This tutorial shows how provider access and sandbox policy work together. You +attach a GitHub provider that contributes read-only GitHub access, observe a +denied push, and add a user policy that permits writes to one repository. -After completing this tutorial, you have: - -- A running sandbox with Claude Code that can push to a GitHub repository. -- A custom network policy that grants GitHub access for a specific repository. -- Experience with the policy iteration workflow: fail, diagnose, update, verify. - - -This tutorial shows example prompts and responses from Claude Code. The exact wording you see might vary between sessions. Use the examples as a guide for the type of interaction, not as expected output. - - +The built-in [default policy](/reference/default-policy) does not grant network +access. Imported provider profiles contribute the endpoints and executable +paths needed by their providers. ## Prerequisites -This tutorial requires the following: - -- A working OpenShell installation. Complete the [Quickstart](/get-started/quickstart) before proceeding. -- A GitHub personal access token (PAT) with `repo` scope. Generate one from the [GitHub personal access token settings page](https://github.com/settings/tokens) by selecting **Generate new token (classic)** and enabling the `repo` scope. -- An [Anthropic account](https://console.anthropic.com/) with access to Claude Code. OpenShell provides the sandbox runtime, not the agent. You must authenticate with your own account. -- A GitHub repository you own to use as the push target. A scratch repository is sufficient. You can [create one](https://github.com/new) with a README if needed. - -This tutorial uses two terminals to demonstrate the iterative policy workflow: +You need: -- **Terminal 1**: The sandbox terminal. You create the sandbox in this terminal by running `openshell sandbox create` and interact with Claude Code inside it. -- **Terminal 2**: A terminal outside the sandbox on your machine. You use this terminal for viewing the sandbox logs with `openshell term` and applying an updated policy with `openshell policy set`. +- A working OpenShell installation and active gateway. +- An OCI image you built with Claude Code, `git`, and the GitHub CLI installed. +- An Anthropic API key. +- A GitHub fine-grained personal access token with read and write access to the + target repository's contents. +- A scratch GitHub repository that you can push to. -Each section below indicates which terminal to use. +Use two host terminals. Terminal 1 runs the agent. Terminal 2 inspects denials +and updates policy. -## Set Up a Sandbox with Your GitHub Token +## Import the Provider Profiles -Depending on whether you start a new sandbox or use an existing sandbox, choose the appropriate tab and follow the instructions. - - - - -In terminal 2, create a new sandbox with Claude Code. The [default policy](/reference/default-policy) is applied automatically, which allows read-only access to GitHub. - -Create a [credential provider](/sandboxes/manage-providers) that injects your GitHub token into the sandbox automatically. The provider reads `GITHUB_TOKEN` from your host environment and sets it as an environment variable inside the sandbox: +Download and review the Claude Code and GitHub profiles. Confirm that their +`binaries` paths match the executables in your image, then import them: ```shell +curl -LsSfO https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/claude-code.yaml curl -LsSfO https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/github.yaml -openshell provider profile import -f github.yaml --global - -GITHUB_TOKEN= -openshell provider create --name my-github --type github --from-existing -openshell sandbox create --from registry.example.com/your-org/claude-agent:latest --provider my-github -- claude +openshell profile import -f claude-code.yaml --global +openshell profile import -f github.yaml --global ``` -The import step registers the `github` profile on your gateway. OpenShell ships -no profiles inside the gateway binary, so `--type github` resolves only after the -profile is imported. Import it once per gateway. +The gateway contains no built-in profiles. Import each profile once per gateway +at the scope where users need it. -`openshell sandbox create` keeps the sandbox running after Claude Code exits, so you can apply policy updates later without recreating the environment. Add `--no-keep` if you want the sandbox deleted automatically instead. +## Create the Providers -Claude Code starts inside the sandbox. It prints an authentication link. Open it in your browser, sign in to your Anthropic account, and return to the terminal. When prompted, trust the `/sandbox` workspace to allow Claude Code to read and write files. - - - - -In terminal 1, connect to a sandbox that is already running and set your GitHub token as an environment variable: +Create provider instances from credentials in your host environment. Passing a +credential on the same command line exports it only to that command: ```shell -openshell sandbox connect -export GITHUB_TOKEN= -``` - -To find the name of running sandboxes, run `openshell sandbox list` in terminal 2. - - - - - -## Push Code to GitHub +ANTHROPIC_API_KEY= \ + openshell provider create --name my-claude --type claude-code --from-existing -In terminal 1, ask Claude Code to write a simple script and push it to your repository. Replace `` with your GitHub organization or username and `` with your repository name. - -```md title="Prompt" wordWrap showLineNumbers={false} -Write a `hello_world.py` script and push it to `https://github.com//`. +GITHUB_TOKEN= \ + openshell provider create --name my-github --type github --from-existing ``` -Claude recognizes that it needs GitHub credentials. It asks how you want to authenticate. Provide your GitHub personal access token by pasting it into the conversation. Claude configures authentication and attempts the push. - -The push fails. Claude reports an error, but the failure is not an authentication problem. The default sandbox policy permits read-only access to GitHub and blocks write operations, so the proxy denies the push before the request reaches the GitHub server. - -## Diagnose the Denial +OpenShell stores the credentials through the configured credential driver. The +agent receives opaque placeholders; the sandbox proxy resolves them only for +endpoints allowed by the corresponding profile. -In this section, you diagnose the denial from your machine and from inside the sandbox. +## Start the Agent -### View the Logs from Your Machine - -In terminal 2, launch the OpenShell terminal: +In terminal 1, start Claude Code from your image and attach both providers: ```shell -openshell term +openshell sandbox create \ + --name github-demo \ + --from registry.example.com/your-org/claude-agent:latest \ + --provider my-claude \ + --provider my-github \ + -- claude ``` -The dashboard shows sandbox status and a live stream of policy decisions. Look for entries with `l7_decision=deny`. Select a deny entry to see the full detail: +To use an existing sandbox, attach the providers from a host terminal, then +start a new agent process so it receives the provider environment variables: -```text -l7_action: PUT -l7_target: /repos///contents/hello_world.py -l7_decision: deny -dst_host: api.github.com -dst_port: 443 -l7_protocol: rest -policy: github_rest_api -l7_deny_reason: PUT /repos///contents/hello_world.py not permitted by policy +```shell +openshell sandbox provider attach my-claude +openshell sandbox provider attach my-github +openshell sandbox connect ``` -The log shows that the sandbox proxy intercepted an outbound `PUT` request to `api.github.com` and denied it. The `github_rest_api` policy allows read operations (GET) but blocks write operations (PUT, POST, DELETE) to the GitHub API. A similar denial appears for `github.com` if Claude attempted a git push over HTTPS. +## Attempt a Push -### Ask Claude Code to Check the Sandbox Logs - -In terminal 1, ask Claude Code to check the sandbox logs for denied requests: +Ask the agent to create a file and push it to your scratch repository. Replace +`` and `` with the repository owner and name: ```md title="Prompt" wordWrap showLineNumbers={false} -Check the sandbox logs for any denied network requests. What is blocking the push? +Create `hello_world.py`, commit it, and push it to +`https://github.com//.git`. Use the GitHub credentials already +available in the environment. Do not ask me to paste a token. ``` -Claude reads the deny entries and identifies the root cause. It explains that the failure is a sandbox network policy restriction, not a token permissions issue. For example, the following is a possible response: - - -The sandbox runs a proxy that enforces policies on outbound traffic. -The `github_rest_api` policy allows GET requests (used to read the file) -but blocks PUT/write requests to GitHub. This is a sandbox-level restriction, -not a token issue. No matter what token you provide, pushes through the API -are blocked until you update the policy. - - -Both perspectives confirm the same thing: the proxy is doing its job. The default policy is designed to be restrictive. To allow GitHub pushes, you need to update the network policy. - -Copy the deny reason from Claude's response. You paste it into an agent running on your machine in the next step. +The push fails. The GitHub profile permits clone and fetch operations but does +not permit `git-receive-pack`, the Git Smart HTTP operation used for a push. +The credential remains attached and endpoint-scoped; changing the token would +not grant the missing network authority. -## Update the Policy from Your Machine +## Inspect the Denial -In terminal 2, paste the deny reason from the previous step into your coding agent on your machine, such as Claude Code or Cursor, and ask it to recommend a policy update. The deny reason gives the agent the context it needs to generate the correct policy rules. After pasting the following prompt sample, properly provide the GitHub organization and repository names of the repository you are pushing to. +In terminal 2, inspect recent sandbox logs: -```md title="Prompt" wordWrap showLineNumbers={false} -Based on the following deny reasons, recommend a sandbox policy update that allows GitHub pushes to `https://github.com//`, and save to `/tmp/sandbox-policy-update.yaml`: - -The `filesystem_policy` and `landlock` sections are static. They are read once at sandbox creation and cannot be changed by a hot reload. They are included here for completeness so the file is self-contained. Process identity is omitted so the active compute driver can select it, and only the `network_policies` section takes effect when you apply this to a running sandbox. +```shell +openshell logs github-demo --level warn --since 5m ``` -The following steps outline the expected process done by the agent: - -1. Inspects the deny reasons. -2. Writes an updated policy that adds `github_git` and `github_api` blocks that grant write access to your repository. -3. Saves the policy to `/tmp/sandbox-policy-update.yaml`. +You should see a denial for a request resembling this one: -## Review the Generated Policy - -Refer to the following policy example to compare with the generated policy before applying it. Confirm that the policy grants only the access you expect. In this case, `git push` operations and GitHub REST API access scoped to a single repository. +```text +action=deny dst_host=github.com dst_port=443 binary=/usr/bin/git l7_action=POST l7_target=//.git/git-receive-pack +``` - +You can also run `openshell term` to inspect policy decisions in the terminal +dashboard. -The following YAML shows a complete policy that extends the [default policy](/reference/default-policy) with GitHub access for a single repository. Replace `` with your GitHub organization or username and `` with your repository name. +## Create a Repository-Scoped Policy -The `filesystem_policy` and `landlock` sections are static. OpenShell reads them at sandbox creation, and a hot reload cannot change them. They are included here for completeness so the file is self-contained. Process identity is omitted so the active compute driver can select it, and only the `network_policies` section takes effect when you apply this to a running sandbox. +Create `github-push.yaml`. Replace `` and ``, and adjust `binaries` +to match your image: ```yaml version: 1 -# ── Static (locked at sandbox creation) ────────────────────────── - filesystem_policy: include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /tmp - - /dev/null + read_only: [/usr, /lib, /proc, /dev/urandom, /etc, /var/log] + read_write: [/tmp, /dev/null] landlock: compatibility: best_effort -# ── Dynamic (hot-reloadable) ───────────────────────────────────── - network_policies: - - # Claude Code ↔ Anthropic API - claude_code: - name: claude-code - endpoints: - - { host: api.anthropic.com, port: 443, protocol: rest, enforcement: enforce, access: full } - - { host: statsig.anthropic.com, port: 443 } - - { host: sentry.io, port: 443 } - - { host: raw.githubusercontent.com, port: 443 } - - { host: platform.claude.com, port: 443 } - binaries: - - { path: /usr/local/bin/claude } - - { path: /usr/bin/node } - - # NVIDIA inference endpoint - nvidia_inference: - name: nvidia-inference - endpoints: - - { host: integrate.api.nvidia.com, port: 443 } - binaries: - - { path: /usr/bin/curl } - - { path: /bin/bash } - - { path: /usr/local/bin/opencode } - - # ── GitHub: git operations (clone, fetch, push) ────────────── - - github_git: - name: github-git + github_repository_push: + name: github-repository-push endpoints: - host: github.com port: 443 @@ -240,126 +154,70 @@ network_policies: method: POST path: "//.git/git-receive-pack" binaries: - - { path: /usr/bin/git } + - path: /usr/bin/git + - path: /usr/local/bin/git - # ── GitHub: REST API ───────────────────────────────────────── - - github_api: - name: github-api + github_repository_api: + name: github-repository-api endpoints: - host: api.github.com port: 443 - path: "/repos///**" protocol: rest enforcement: enforce rules: - # Full read-write access to the repository - allow: method: "*" path: "/repos///**" - - host: api.github.com - port: 443 - path: "/graphql" - protocol: graphql - enforcement: enforce - rules: - # GitHub GraphQL API (used by gh CLI) - - allow: - operation_type: query - - allow: - operation_type: mutation - fields: [createIssue, updateIssue, addComment] - deny_rules: - - operation_type: mutation - fields: [deleteRepository, deleteRef, updateBranchProtectionRule] - binaries: - - { path: /usr/local/bin/claude } - - { path: /usr/local/bin/opencode } - - { path: /usr/bin/gh } - - { path: /usr/bin/curl } - - # ── Package managers ───────────────────────────────────────── - - pypi: - name: pypi - endpoints: - - { host: pypi.org, port: 443 } - - { host: files.pythonhosted.org, port: 443 } - - { host: github.com, port: 443 } - - { host: objects.githubusercontent.com, port: 443 } - - { host: api.github.com, port: 443 } - - { host: downloads.python.org, port: 443 } - binaries: - - { path: /sandbox/.venv/bin/python } - - { path: /sandbox/.venv/bin/python3 } - - { path: /sandbox/.venv/bin/pip } - - { path: "/sandbox/.uv/python/**/python*" } - - { path: /usr/local/bin/uv } - - { path: "/sandbox/.uv/python/**" } - - # ── VS Code Remote ────────────────────────────────────────── - - vscode: - name: vscode - endpoints: - - { host: update.code.visualstudio.com, port: 443 } - - { host: "*.vo.msecnd.net", port: 443 } - - { host: vscode.download.prss.microsoft.com, port: 443 } - - { host: marketplace.visualstudio.com, port: 443 } - - { host: "*.gallerycdn.vsassets.io", port: 443 } binaries: - - { path: /usr/bin/curl } - - { path: /usr/bin/wget } - - { path: "/sandbox/.vscode-server/**" } - - { path: "/sandbox/.vscode-remote-containers/**" } + - path: /usr/bin/gh + - path: /usr/local/bin/gh ``` -The following table summarizes the two GitHub-specific blocks: - -| Block | Endpoint | Behavior | -|---|---|---| -| `github_git` | `github.com:443` | Git Smart HTTP protocol. The proxy auto-detects and terminates TLS to inspect requests. Permits `info/refs` (clone/fetch), `git-upload-pack` (fetch data), and `git-receive-pack` (push) for the specified repository. Denies all operations on unlisted repositories. | -| `github_api` | `api.github.com:443` | REST API. The proxy auto-detects and terminates TLS to inspect requests. Permits all HTTP methods for the specified repository and GraphQL queries. Denies API access to unlisted repositories. | +The first entry grants Git Smart HTTP operations only for the selected +repository. The second lets `gh` use REST operations scoped to the same +repository. The attached GitHub provider continues to supply credential +placement and its broader read-only rules. -The remaining blocks (`claude_code`, `nvidia_inference`, `pypi`, `vscode`) are identical to the [default policy](/reference/default-policy). The default policy's `github_ssh_over_https` and `github_rest_api` blocks are replaced by the `github_git` and `github_api` blocks above, which grant write access to the specified repository. Sandbox behavior outside of GitHub operations is unchanged. - -For details on policy block structure, refer to [Policies](/sandboxes/policies). - +The filesystem and Landlock sections preserve the fallback policy's static +settings because `policy set` replaces the complete user-authored base policy. +Process identity remains omitted so the compute driver can select it. ## Apply the Policy -After you have reviewed the generated policy, apply it to the running sandbox: +Apply the policy and wait for the new revision to load: ```shell -openshell policy set --policy /tmp/sandbox-policy-update.yaml --wait +openshell policy set github-demo --policy github-push.yaml --wait ``` -Network policies are hot-reloadable. The `--wait` flag blocks until the policy engine confirms the new revision loaded, and the update takes effect immediately without restarting the sandbox or reconnecting Claude Code. +Network policy changes hot-reload without recreating the sandbox. Provider +rules are composed with this user-authored base policy. -## Retry the Push +## Retry and Verify -In terminal 1, ask Claude Code to retry the push: +Ask the agent to retry the push. Then confirm that the proxy allowed the scoped +request: -```md title="Prompt" wordWrap showLineNumbers={false} -The sandbox policy has been updated. Try pushing to the repository again. +```shell +openshell logs github-demo --since 5m +openshell policy get github-demo --full ``` -The push completes successfully. The `openshell term` dashboard now shows `l7_decision=allow` entries for `api.github.com` and `github.com` where it previously showed denials. +The push succeeds for the selected repository. Requests to push to another +repository remain denied. ## Clean Up -When you are finished, delete the sandbox to free gateway compute resources: +Delete the sandbox when you are finished: ```shell -openshell sandbox delete +openshell sandbox delete github-demo ``` ## Next Steps -The following resources cover related topics in greater depth: - -- To add per-repository access levels (read-write vs read-only) or restrict to specific API methods, refer to the [Policy Schema Reference](/reference/policy-schema). -- To learn the full policy iteration workflow (pull, edit, push, verify), refer to [Policies](/sandboxes/policies). -- To inject credentials automatically instead of pasting tokens, refer to [Manage Providers](/sandboxes/manage-providers) +- Review [Profiles](/providers/profiles) for endpoint-scoped credential placement. +- Review [Policies](/sandboxes/policies) for incremental policy updates and policy history. +- Review the [Policy Schema](/reference/policy-schema) for REST, GraphQL, and other protocol rules. diff --git a/docs/get-started/tutorials/index.mdx b/docs/get-started/tutorials/index.mdx index cda35bfffd..8d13198d99 100644 --- a/docs/get-started/tutorials/index.mdx +++ b/docs/get-started/tutorials/index.mdx @@ -27,19 +27,4 @@ Launch Claude Code in a sandbox, diagnose a policy denial, and iterate on a cust Configure a Microsoft Graph provider profile with gateway-managed OAuth2 refresh-token rotation. - - -Attach a profile that authorizes a sandbox to use a host-level Ollama service. - - - - -Attach an endpoint-bearing profile and call a local LM Studio server directly. - - - - -Run the OpenShell gateway as a Docker Compose service and create agent sandboxes including OpenClaw. - - diff --git a/docs/get-started/tutorials/inference-ollama.mdx b/docs/get-started/tutorials/inference-ollama.mdx deleted file mode 100644 index 017e8ec889..0000000000 --- a/docs/get-started/tutorials/inference-ollama.mdx +++ /dev/null @@ -1,99 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Run Local Inference with Ollama" -sidebar-title: "Inference with Ollama" -slug: "get-started/tutorials/inference-ollama" -description: "Authorize an OpenShell sandbox to use an Ollama service running on the gateway host." -keywords: "Generative AI, Cybersecurity, Tutorial, Ollama, Local Inference, Sandbox, Provider Profile" ---- - -This tutorial connects an explicitly authorized sandbox to an Ollama service running on the same machine as the OpenShell gateway. - -## Prerequisites - -Complete the [Quickstart](/get-started/quickstart) before proceeding. Your workload image must contain the client or agent you plan to use. - -## Start Ollama on a Reachable Address - -Install Ollama on the gateway host and listen on an address reachable from sandboxes: - -```shell -curl -fsSL https://ollama.com/install.sh | sh -OLLAMA_HOST=0.0.0.0:11434 ollama serve -``` - -In another terminal, pull a model: - -```shell -ollama pull qwen3.5:0.8b -``` - -## Import an Endpoint-bearing Profile - -Save this as `ollama-openai.yaml`: - -```yaml -id: ollama-openai -display_name: Ollama -description: Host-level Ollama OpenAI-compatible API -category: inference -inference_capable: true -credentials: [] -endpoints: - - host: host.openshell.internal - port: 11434 - protocol: rest - access: read-write - enforcement: enforce -binaries: - - /usr/bin/curl - - /usr/local/bin/curl - - /usr/bin/python3 - - /usr/local/bin/python - - /sandbox/.uv/python/** - - /sandbox/.venv/** -``` - -```shell -openshell provider profile lint -f ollama-openai.yaml -openshell provider profile import -f ollama-openai.yaml -openshell provider create --name ollama --type ollama-openai -``` - -The provider has no secret because this Ollama server does not authenticate. Its attachment still carries the endpoint and binary policy. - -## Attach and Verify - -Use an explicit workload image containing `curl`: - -```shell -openshell sandbox create \ - --name ollama-client \ - --from registry.example.com/tools/curl:latest \ - --provider ollama \ - -- \ - curl http://host.openshell.internal:11434/v1/chat/completions \ - --json '{ - "model":"qwen3.5:0.8b", - "messages":[{"role":"user","content":"hello"}], - "max_tokens":10 - }' -``` - -For an OpenAI SDK client, set its base URL to `http://host.openshell.internal:11434/v1`, use any non-empty API key value the SDK accepts, and select the real Ollama model in the request. - -## Troubleshooting - -- Bind host-level Ollama to `0.0.0.0`, not `127.0.0.1`. -- Use `host.openshell.internal`, not `localhost`, from a sandbox. -- Confirm attachment with `openshell sandbox provider list ollama-client`. -- Inspect the effective policy with `openshell policy get ollama-client --full`. -- Run `ollama ps` and `ollama pull ` when the model is unavailable. -- If the gateway is remote, the hostname refers to the remote gateway host, not your laptop. Use a shared service address or tunnel instead. - -## Next Steps - -- [Provider-backed Inference](/sandboxes/inference-routing) -- [Profiles](/providers/profiles) -- [Customize Sandbox Policies](/sandboxes/policies) diff --git a/docs/get-started/tutorials/local-inference-lmstudio.mdx b/docs/get-started/tutorials/local-inference-lmstudio.mdx deleted file mode 100644 index 4f7ffc7aed..0000000000 --- a/docs/get-started/tutorials/local-inference-lmstudio.mdx +++ /dev/null @@ -1,139 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Use Local Inference with LM Studio" -sidebar-title: "Local Inference with LM Studio" -slug: "get-started/tutorials/local-inference-lmstudio" -description: "Attach a provider profile for a local LM Studio server and call its native compatible endpoints." -keywords: "Generative AI, Cybersecurity, Tutorial, LM Studio, Local Inference, Sandbox, Provider Profile" ---- - -This tutorial exposes an LM Studio server on the gateway host to selected -sandboxes. The provider attachment grants access; the workload calls LM -Studio's OpenAI- or Anthropic-compatible endpoint directly. - -## Prerequisites - -- Complete the [Quickstart](/get-started/quickstart). -- Install [LM Studio](https://lmstudio.ai/download) on the gateway host. - -For a headless installation: - - - - -```shell -curl -fsSL https://lmstudio.ai/install.sh | bash -``` - - - - -```shell -irm https://lmstudio.ai/install.ps1 | iex -``` - - - - -Start the daemon and server: - -```shell -lms daemon up -lms server start --bind 0.0.0.0 -``` - -Load a model: - -```shell -lms get qwen/qwen3.5-2b -lms load qwen/qwen3.5-2b -``` - -## Create the Provider Profile - -Save this as `lmstudio.yaml`: - -```yaml -id: lmstudio -display_name: LM Studio -description: Host-level LM Studio compatible APIs -category: inference -inference_capable: true -credentials: [] -endpoints: - - host: host.openshell.internal - port: 1234 - protocol: rest - access: read-write - enforcement: enforce -binaries: - - /usr/bin/curl - - /usr/local/bin/curl - - /usr/bin/python3 - - /usr/local/bin/python - - /sandbox/.uv/python/** - - /sandbox/.venv/** -``` - -```shell -openshell provider profile lint -f lmstudio.yaml -openshell provider profile import -f lmstudio.yaml -openshell provider create --name lmstudio --type lmstudio -``` - -The profile intentionally has no credential. LM Studio does not require one by -default, but the provider attachment still supplies its policy boundary. - -## Attach and Call the Native Endpoint - -OpenAI-compatible request: - -```shell -openshell sandbox create \ - --name lmstudio-client \ - --provider lmstudio \ - -- \ - curl http://host.openshell.internal:1234/v1/chat/completions \ - --json '{ - "model":"qwen/qwen3.5-2b", - "messages":[{"role":"user","content":"hello"}], - "max_tokens":10 - }' -``` - -Anthropic-compatible request: - -```shell -openshell sandbox exec lmstudio-client -- \ - curl http://host.openshell.internal:1234/v1/messages \ - --json '{ - "model":"qwen/qwen3.5-2b", - "messages":[{"role":"user","content":"hello"}], - "max_tokens":10 - }' -``` - -Configure SDKs with the same native base URL. Some SDKs require a non-empty -API key even when LM Studio ignores it; use a literal non-secret value such as -`unused`. Keep the actual model ID in the client request. - -## Troubleshooting - -- Enable **Serve on Local Network** in the LM Studio Developer tab, or use - `lms server start --bind 0.0.0.0`. -- Use `host.openshell.internal`, not `127.0.0.1` or `localhost`, in the - sandbox. -- Confirm the model is loaded with `lms ps`. -- Confirm the attachment with - `openshell sandbox provider list lmstudio-client`. -- Inspect endpoint and binary policy with - `openshell policy get lmstudio-client --full`. -- A remote gateway cannot reach an LM Studio process on your laptop without a - tunnel or shared network path. - -## Next Steps - -- [Provider-backed Inference](/sandboxes/inference-routing) -- [Profiles](/providers/profiles) -- [LM Studio CLI](https://lmstudio.ai/docs/cli) diff --git a/docs/get-started/tutorials/microsoft-graph-provider-refresh.mdx b/docs/get-started/tutorials/microsoft-graph-provider-refresh.mdx index 4204e7ebfb..301678d91f 100644 --- a/docs/get-started/tutorials/microsoft-graph-provider-refresh.mdx +++ b/docs/get-started/tutorials/microsoft-graph-provider-refresh.mdx @@ -22,7 +22,7 @@ This tutorial starts after your OAuth client has already completed the initial M ## Prerequisites -- A working OpenShell installation with an active gateway. Complete the [Quickstart](/get-started/quickstart) before proceeding. +- A working OpenShell installation with an active gateway. Refer to [Installation](/about/installation) before proceeding. - A Microsoft Entra app registration that can acquire delegated Microsoft Graph mail access. - Delegated Microsoft Graph mail permission for the signed-in user. `Mail.Read` allows reading the signed-in user's mailbox; see the [Microsoft Graph permissions reference](https://learn.microsoft.com/en-us/graph/permissions-reference). OAuth material from your initial Microsoft sign-in flow: @@ -152,6 +152,7 @@ Launch a sandbox with the Microsoft Graph provider attached: ```shell openshell sandbox create \ --name microsoft-graph-mail \ + --from registry.example.com/team/agent-tools:1.0 \ --provider microsoft-mail \ --no-auto-providers \ -- /bin/sh diff --git a/docs/index.mdx b/docs/index.mdx index 9a40bca095..a48bff2bca 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -49,12 +49,13 @@ Install OpenShell and create your first sandbox in two commands. ```shell curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh -openshell sandbox create --name quickstart +openshell sandbox create ``` -Refer to the [Quickstart](/get-started/quickstart) for more details. +Refer to [Run Your First Agent](/about/run-an-agent) for the complete image, +provider, and policy workflow. --- @@ -70,9 +71,9 @@ Learn about OpenShell and its capabilities. Concept - + -Install OpenShell and create your first sandbox in two commands. +Prepare an image, attach providers, and launch an agent in a sandbox. Tutorial @@ -91,7 +92,7 @@ Deploy gateways, create sandboxes, configure policies, providers, and workload i Concept - + Attach model providers to selected sandboxes and call their native endpoints without exposing credentials. @@ -105,9 +106,9 @@ Understand sandbox logs, access them with the CLI and TUI, and export OCSF JSON How-To - + -Policy schema, environment variables, and default policy details. +Define filesystem, process, and network controls for sandbox workloads. Reference diff --git a/docs/index.yml b/docs/index.yml index 45db451a78..30241862fb 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -6,28 +6,115 @@ landing-page: path: index.mdx navigation: -- folder: about - title: "About NVIDIA OpenShell" -- section: "Get Started" - slug: get-started +- section: "About NVIDIA OpenShell" + slug: about contents: - - page: "Quickstart" - path: get-started/quickstart.mdx - - folder: get-started/tutorials - skip-slug: true -- folder: sandboxes - title: "Manage OpenShell" -- folder: providers - title: "Providers" -- folder: extensibility - title: "Extensibility" + - page: "Why OpenShell" + path: about/overview.mdx + - page: "Architecture" + path: about/how-it-works.mdx + - page: "Installation" + path: about/installation.mdx + - page: "Run Your First Agent" + path: about/run-an-agent.mdx + - page: "Support Matrix" + path: reference/support-matrix.mdx +- section: "How It Works" + slug: manage + contents: + - section: "Gateways" + slug: gateways + contents: + - page: "Overview" + path: sandboxes/manage-gateways.mdx + - page: "Authentication" + path: reference/gateway-auth.mdx + - page: "Configuration" + path: reference/gateway-config.mdx + - page: "Container Deployment" + path: reference/container-gateway.mdx + - page: "Workspaces" + path: sandboxes/manage-workspaces.mdx + - section: "Sandboxes" + slug: sandboxes + contents: + - page: "Overview" + path: sandboxes/manage-sandboxes.mdx + - page: "Runtimes" + path: reference/sandbox-compute-drivers.mdx + - section: "Providers" + slug: providers + contents: + - page: "Overview" + path: sandboxes/manage-providers.mdx + - page: "Profiles" + path: providers/profiles.mdx + - page: "AWS" + path: providers/aws-sigv4.mdx + - page: "Google" + path: providers/google-cloud.mdx + - section: "Policies" + slug: policies + contents: + - page: "Overview" + path: sandboxes/policies.mdx + - page: "Advisor" + path: sandboxes/policy-advisor.mdx + - page: "Schema" + path: reference/policy-schema.mdx + - page: "Prover" + path: reference/policy-prover.mdx + - page: "Default Policy" + path: reference/default-policy.mdx + - page: "Inference" + path: sandboxes/inference-routing.mdx +- section: "Extensibility" + slug: extensibility + contents: + - page: "Overview" + path: extensibility/extension-negotiation.mdx + - page: "Middleware" + path: extensibility/supervisor-middleware.mdx + - page: "Interceptors" + path: extensibility/gateway-interceptors.mdx + - page: "Drivers" + path: extensibility/drivers.mdx + - page: "Isolation Backends" + path: extensibility/isolation-backends.mdx - folder: observability title: "Observability" - folder: kubernetes title: "Kubernetes" -- folder: reference - title: "Reference" +- section: "Tutorials" + slug: tutorials + contents: + - page: "First Network Policy" + path: get-started/tutorials/first-network-policy.mdx + - page: "GitHub Push Access" + path: get-started/tutorials/github-sandbox.mdx + - page: "Microsoft Graph Provider Refresh" + path: get-started/tutorials/microsoft-graph-provider-refresh.mdx +- section: "SDK Reference" + slug: sdk + contents: + - page: "Go" + path: sdk/go.mdx + - page: "Rust" + path: sdk/rust.mdx + - page: "Python" + path: sdk/python.mdx + - page: "TypeScript" + path: sdk/typescript.mdx + - page: "API Errors" + path: reference/api-errors.mdx + - page: "Protobuf Time Types" + path: reference/protobuf-time-types.mdx - folder: security title: "Security" +- section: "Upgrade Guides" + slug: upgrade + contents: + - page: "0.1.0" + path: upgrade/0-1-0.mdx - folder: resources title: "Resources" diff --git a/docs/kubernetes/access-control.mdx b/docs/kubernetes/access-control.mdx index 1ff54a0e01..3d17597fbe 100644 --- a/docs/kubernetes/access-control.mdx +++ b/docs/kubernetes/access-control.mdx @@ -5,7 +5,7 @@ title: "Access Control" sidebar-title: "Access Control" description: "Configure OIDC user authentication or reverse-proxy auth termination for a Kubernetes-deployed OpenShell gateway." keywords: "Generative AI, Cybersecurity, Kubernetes, Authentication, mTLS, OIDC, Keycloak, Entra ID, Okta, Gateway Auth" -position: 5 +position: 6 --- The OpenShell gateway supports two access-control models for human callers on Kubernetes: diff --git a/docs/kubernetes/high-availability.mdx b/docs/kubernetes/high-availability.mdx new file mode 100644 index 0000000000..c45ce9a1f6 --- /dev/null +++ b/docs/kubernetes/high-availability.mdx @@ -0,0 +1,215 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "High Availability" +sidebar-title: "High Availability" +description: "Run multiple OpenShell gateway replicas on Kubernetes with shared PostgreSQL and authenticated peer routing." +keywords: "Generative AI, Cybersecurity, Kubernetes, High Availability, HA, Gateway, PostgreSQL, Replicas, Failover" +position: 3 +--- + +Run two or more gateway replicas when the OpenShell control plane must remain +available during a gateway pod failure or rolling update. Every replica can +serve API requests. The replicas share persistent state through PostgreSQL and +relay session-bound requests to the replica connected to each sandbox +supervisor. + + +Gateway replicas do not make PostgreSQL highly available. Use a managed +PostgreSQL service or a separately operated PostgreSQL cluster with its own +backup and failover plan. + + +## Requirements + +An HA gateway deployment requires: + +- Two or more schedulable Kubernetes nodes or failure domains. +- PostgreSQL reachable from every gateway pod. +- A Kubernetes Secret in the OpenShell namespace with the PostgreSQL connection + URI in a key named `uri`. +- `workload.kind: deployment`. The default StatefulSet and per-pod SQLite + database are intended for a single gateway replica. +- An ingress or load balancer that routes clients to the gateway Service. Refer + to [Ingress](/kubernetes/ingress) for a Gateway API configuration. + +The Helm chart rejects `replicaCount` values above `1` unless +`server.externalDbSecret` is set. It also rejects a multi-replica StatefulSet +unless `workload.allowMultiReplicaStatefulSet` is explicitly enabled. Use a +Deployment for the standard HA configuration. + +## Create the PostgreSQL Secret + +Provision PostgreSQL outside the OpenShell chart, then create the connection +Secret. The chart does not install PostgreSQL. + +```shell +kubectl create namespace openshell + +kubectl -n openshell create secret generic openshell-postgres \ + --from-literal=uri='postgresql://openshell:@:5432/openshell' +``` + +Use your PostgreSQL provider's required TLS parameters in the connection URI. +Keep the Secret in the same namespace as the gateway release. + +## Configure Gateway Replicas + +Create `values-ha.yaml`: + +```yaml +replicaCount: 2 + +workload: + kind: deployment + +server: + externalDbSecret: openshell-postgres + +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: openshell + app.kubernetes.io/instance: openshell + topologyKey: kubernetes.io/hostname +``` + +The anti-affinity rule places the two replicas on different nodes. The +`app.kubernetes.io/instance` value must match the Helm release name. Change the +topology key to a zone label when replicas must span availability zones. A +required rule leaves a replica Pending when the cluster does not have enough +matching failure domains; use preferred anti-affinity when scheduling the +replica is more important than strict separation. + +Install or upgrade OpenShell: + +```shell +helm upgrade --install openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --values values-ha.yaml \ + --wait +``` + +The chart shares its retained sandbox JWT signing material, TLS material, and +default credential-storage encryption key across the gateway pods. Do not +create different copies of those Secrets for individual replicas. + +## Verify the Deployment + +Wait for all gateway replicas to become ready: + +```shell +kubectl -n openshell rollout status deployment/openshell +kubectl -n openshell get pods \ + -l app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell \ + -o wide +``` + +Confirm that the client-facing Service and headless peer Service have ready +endpoints: + +```shell +kubectl -n openshell get service openshell openshell-peer +kubectl -n openshell get endpointslice \ + -l app.kubernetes.io/service-name=openshell +kubectl -n openshell get endpointslice \ + -l app.kubernetes.io/service-name=openshell-peer +``` + +If you changed the chart's generated names with `nameOverride` or +`fullnameOverride`, use the rendered Service names instead. + +After registering the gateway with the CLI, verify that requests succeed: + +```shell +openshell status +openshell sandbox list +``` + +## Protect Voluntary Disruptions + +Add a PodDisruptionBudget so a voluntary disruption, such as node maintenance, +does not evict every ready gateway pod at once: + +```yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: openshell + namespace: openshell +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: openshell + app.kubernetes.io/instance: openshell +``` + +Apply the manifest separately from the OpenShell chart. Match the namespace and +release labels to your installation. A PodDisruptionBudget controls voluntary +evictions; it does not protect against an unavailable node or failure domain. + +## How Peer Routing Works + +A sandbox supervisor keeps one active control stream to one gateway replica. +That replica records ownership in PostgreSQL. When Kubernetes sends a client +request to another replica, the receiving gateway relays exec, forwarding, +file-sync, service, and status traffic to the owner through the headless peer +Service. + +Peer RPCs use a projected, pod-bound ServiceAccount token with the +`openshell-gateway-peer` audience. The receiving gateway validates the token +with the Kubernetes TokenReview API, verifies the live pod identity and release +labels, and authorizes only peer methods. When gateway TLS is enabled, peer +connections also use the chart's TLS materials. + +You do not need session affinity at the ingress or client-facing Service. + +## Failure and Rollout Behavior + +When a non-owner gateway pod stops, Kubernetes removes it from ready Service +endpoints and new requests go to another replica. Existing requests through the +stopped pod disconnect and must be retried. + +When an owner gateway pod stops, its connected supervisors reconnect through +the gateway Service and publish new ownership. Other replicas re-read ownership +while waiting to relay a request. A request fails if no fresh, reachable owner +appears before its normal deadline. + +Established streams do not move between gateway replicas. An interrupted exec, +forwarding, or service stream must reconnect. File sync retries with a new SSH +session instead of resuming the interrupted byte stream. Rolling updates can +temporarily concentrate supervisor sessions on the replicas that stayed up; +client requests remain routable through peer relay. + +## Scale the Gateway + +Change `replicaCount` in `values-ha.yaml`, then apply the release again: + +```shell +helm upgrade openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --values values-ha.yaml \ + --wait +``` + +Keep at least two ready replicas when availability must survive one gateway pod +failure. Size PostgreSQL connection capacity and the cluster nodes for the +selected replica count. + +## Next Steps + +- To expose the gateway through a highly available data path, refer to + [Ingress](/kubernetes/ingress). +- To configure automatic certificate renewal, refer to + [Managing Certificates](/kubernetes/managing-certificates). +- To configure user authentication and authorization, refer to + [Access Control](/kubernetes/access-control). diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 5af0589f20..fd6be78fbd 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -5,7 +5,7 @@ title: "Ingress" sidebar-title: "Ingress" description: "Expose the OpenShell gateway externally using the Kubernetes Gateway API and a GRPCRoute." keywords: "Generative AI, Cybersecurity, Kubernetes, Gateway API, Envoy Gateway, GRPCRoute, Ingress, External Access" -position: 4 +position: 5 --- By default, the OpenShell gateway is only reachable inside the cluster. To let CLI clients connect without a `kubectl port-forward`, expose the gateway through an ingress. diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index efef147f0a..13815e5cf9 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -5,7 +5,7 @@ title: "Managing Certificates" sidebar-title: "Managing Certificates" description: "Configure the OpenShell Helm chart to use cert-manager for mTLS certificate issuance and automatic renewal." keywords: "Generative AI, Cybersecurity, Kubernetes, cert-manager, PKI, TLS, mTLS, Certificates" -position: 3 +position: 4 --- The OpenShell gateway uses mTLS certificates for transport between the gateway and sandbox supervisors. These certificates are not Kubernetes user authentication; configure OIDC or a trusted access proxy for user access. The Helm chart supports two ways to provision and manage the certificate bundle: diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index 39b0a4bf14..e510fc44c4 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -5,7 +5,7 @@ title: "OpenShift" sidebar-title: "OpenShift" description: "Install the OpenShell Helm chart on OpenShift with capability-free sandbox workloads." keywords: "Generative AI, Cybersecurity, Kubernetes, OpenShift, SCC, Security Context Constraints, Helm, Gateway, Installation" -position: 6 +position: 7 --- The Kubernetes driver resolves the UID range assigned to each OpenShift diff --git a/docs/kubernetes/sandbox-runtime.mdx b/docs/kubernetes/sandbox-runtime.mdx index ffa7350f2e..1376916b44 100644 --- a/docs/kubernetes/sandbox-runtime.mdx +++ b/docs/kubernetes/sandbox-runtime.mdx @@ -110,7 +110,10 @@ The driver creates each sandbox generation in a fail-closed order: 3. Inspect the admitted Pod identity, security context, DNS settings, and generation-specific Secret reference. 4. Create a gated supervisor Pod and separate immutable Secrets for sandbox and - supervisor trust material. + supervisor trust material. Outside shared mode, the supervisor Secret also + carries the gateway client TLS material. In managed mode, the driver also + creates immutable copies of the configured image-pull Secrets for the + generation. 5. Remove both scheduling gates. 6. Publish readiness only after the supervisor attaches, confirms enforcement, and registers the gateway relay. @@ -127,11 +130,11 @@ while preserving the workspace PVC. The namespace-wide workload fence remains in place across sandbox generations. If provisioning is interrupted, recovery first confirms that the old workload -Pod is gone. It then lists generation Secrets by sandbox and component labels, -deletes each stale Secret with a UID precondition, and clears the rollback state -so the next generation can be created. The Helm chart grants the gateway -`create`, `list`, and `delete` access to Secrets for this lifecycle. Listing is -required even when no generation Secrets remain. +Pod is gone. It then deletes the recorded generation's Secrets by name and +clears the rollback state so the next generation can be created. Each +generation Secret is owned by its Pod, so Kubernetes garbage collection removes +any other generation. The Helm chart grants the gateway `create` and `delete` +access to Secrets for this lifecycle. ## Check Cluster Requirements diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 4a3539c450..95c3e6c620 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -52,7 +52,7 @@ The chart does not install or upgrade the cluster-scoped Agent Sandbox CRDs or controller. -**Air-gapped clusters:** mirror the manifest above and the `registry.k8s.io/agent-sandbox/agent-sandbox-controller` image referenced inside it to your internal registry, then point the manifest's image reference at your mirror before applying. You will also need to mirror the OpenShell gateway and sandbox images — see the chart's `image.repository` value for the gateway and `server.sandboxImage` / `server.supervisorImage` for the sandbox runtime. +**Air-gapped clusters:** mirror the manifest above and the `registry.k8s.io/agent-sandbox/agent-sandbox-controller` image referenced inside it to your internal registry, then point the manifest's image reference at your mirror before applying. Mirror the OpenShell gateway, supervisor, and trusted sandbox runtime images, then set `global.image.registry` to the mirror. Configure the separate default workload sandbox image with `sandbox.image` when needed. Confirm the controller pod is running before proceeding: @@ -130,13 +130,36 @@ release namespace. For one pre-provisioned namespace, keep `server.sandboxNamespace=app-a`. To map multiple workspaces to separately provisioned namespaces, use `workspaceMode=operator`, configure exactly one of `operatorNamespaceLabel` or `operatorNamespaceFile`, and install the workspace -chart in every allowlisted namespace. +chart in every allowlisted namespace. In operator mode, the workspace chart +Role is the gateway's only Secret permission grant in that namespace. If you enable `server.drivers.kubernetes.allowDriverConfig` on the gateway, also set `gateway.allowDriverConfig=true` on every workspace release. This grants the namespace-scoped PVC metadata read needed to admit caller-selected PVC mounts. Leave both values disabled when callers do not need driver config. +### Store provider credentials in Kubernetes Secrets + +To store provider credentials as Kubernetes Secrets instead of in the gateway +database, enable the Kubernetes Secrets credential driver with a dedicated +namespace: + +```shell +helm upgrade --install openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set server.credentialDrivers.kubernetesSecrets.enabled=true \ + --set server.credentialDrivers.kubernetesSecrets.namespace=openshell-credentials \ + --set server.credentialDrivers.kubernetesSecrets.createNamespace=true +``` + +The driver stores every credential in that namespace, in every workspace mode. +The gateway receives Secret permissions through a Role in that namespace only. For +the full set of options, refer to the +[gateway configuration reference](/reference/gateway-config#credential-drivers). + ## Wait for the gateway to be ready ```shell @@ -196,7 +219,8 @@ The most commonly changed values are: | Value | Purpose | |---|---| -| `image.repository` / `image.tag` | Gateway container image. Defaults to `ghcr.io/nvidia/openshell/gateway:latest`. | +| `global.image.registry` / `global.image.tag` / `global.image.pullPolicy` | Shared registry, tag, and pull policy for the gateway, supervisor, and trusted sandbox runtime. Individual image settings take precedence. | +| `gateway.image.registry` / `gateway.image.repository` / `gateway.image.tag` / `gateway.image.digest` | Gateway container image. Defaults to `ghcr.io/nvidia/openshell/gateway:latest`. | | `replicaCount` | Number of gateway replicas. Values above `1` require shared PostgreSQL through `server.externalDbSecret`. | | `workload.kind` | Gateway workload controller. Use `statefulset` for SQLite or `deployment` with `server.externalDbSecret`. | | `workload.allowMultiReplicaStatefulSet` | Allow `replicaCount > 1` with `workload.kind=statefulset`. Prefer Deployment for external database-backed multi-replica gateways. | @@ -204,7 +228,9 @@ The most commonly changed values are: | `workspaceResources.enabled` | Create namespace-scoped sandbox prerequisites from the gateway chart. Disable when installing the workspace chart separately. | | `server.externalDbSecret` | Secret containing a PostgreSQL connection URI in the `uri` key. Use when the database is managed outside the chart. | | `server.telemetryEnabled` | Enable anonymous OpenShell telemetry from the gateway and its sandbox supervisors. Set to `false` to opt out. | -| `server.sandboxImage` | Default sandbox image used when a sandbox does not specify one. | +| `sandbox.image.repository` / `sandbox.image.tag` / `sandbox.image.digest` | Default sandbox image used when a sandbox does not specify one. | +| `sandboxRuntime.image.registry` / `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` / `sandboxRuntime.image.digest` | Trusted workload-side image that provides the `openshell-sandbox` binary. A digest takes precedence over the tag. | +| `supervisor.image.registry` / `supervisor.image.repository` / `supervisor.image.tag` / `supervisor.image.digest` | Trusted control-side supervisor image. | | `server.sandboxImagePullSecrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | `server.grpcEndpoint` | Endpoint that sandbox supervisors use to call back to the gateway. Must be reachable from inside the cluster. | | `server.disableTls` | Run the gateway over plaintext HTTP. Use only behind a trusted transport. | @@ -217,21 +243,9 @@ The most commonly changed values are: ### Run multiple gateway replicas -Use a Deployment and shared PostgreSQL for high availability: - -```yaml -replicaCount: 2 -workload: - kind: deployment -server: - externalDbSecret: openshell-postgres -``` - -The chart creates the peer Service, projected ServiceAccount identity, and RBAC -that gateway replicas use to route exec, forwarding, file-sync, and service -traffic to the replica holding the external supervisor's live session. Peer -traffic uses the chart TLS materials when gateway TLS is enabled. Do not use -the per-pod SQLite database with more than one replica. +Use a Deployment and shared PostgreSQL to run more than one gateway replica. +Refer to [High Availability](/kubernetes/high-availability) for the database +Secret, Helm values, pod placement, verification, and failover behavior. Use a values file for repeatable deployments: @@ -301,6 +315,19 @@ The chart creates the following RBAC resources in the release namespace: | Role + RoleBinding | Namespace | `openshell-sandbox` | | ClusterRole + ClusterRoleBinding | Cluster | `openshell-node-reader` | +When the Kubernetes Secrets credential driver is enabled, the chart also creates +an `openshell-credential-secrets` Role and RoleBinding in the credential +namespace. That Role grants `get`, `create`, `patch`, and `delete` on all Secrets +in the credential namespace, and is the gateway's only permission grant for provider +credential Secrets. + +In managed and operator workspace modes, the chart creates an +`openshell-workspace-secret-source` Role and RoleBinding in the sandbox +namespace. That Role grants `get` on only the gateway client TLS Secret and, in +managed mode, the configured image-pull Secrets. The gateway stages their +contents into each sandbox runtime generation's Secrets in the workspace +namespace. + The namespaced Role covers sandbox lifecycle and identity: | API Group | Resource | Verbs | @@ -340,6 +367,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency ## Next Steps - Kubernetes sandboxes use separate workload and directly managed supervisor Pods; refer to [Sandbox runtime](/kubernetes/sandbox-runtime). +- To run multiple gateway replicas, refer to [High Availability](/kubernetes/high-availability). - To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress). - To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/providers/aws-sigv4.mdx b/docs/providers/aws-sigv4.mdx index 101d796e5c..a9251f134d 100644 --- a/docs/providers/aws-sigv4.mdx +++ b/docs/providers/aws-sigv4.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "AWS SigV4 Credential Signing" -sidebar-title: "AWS SigV4" +sidebar-title: "AWS" description: "Configure proxy-side AWS SigV4 request signing so sandbox agents can reach AWS services through CONNECT tunnels without holding real credentials." keywords: "Generative AI, Cybersecurity, AI Agents, AWS, SigV4, Bedrock, S3, Credential Signing, Sandbox" --- diff --git a/docs/providers/google-cloud.mdx b/docs/providers/google-cloud.mdx index 9168be25e3..dee210b77b 100644 --- a/docs/providers/google-cloud.mdx +++ b/docs/providers/google-cloud.mdx @@ -1,9 +1,9 @@ --- # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Google Cloud" -sidebar-title: "Google Cloud" -description: "Authenticate with GCP APIs inside OpenShell sandboxes." +title: "Google" +sidebar-title: "Google" +description: "Authenticate with Google Cloud APIs and Vertex AI inside OpenShell sandboxes." keywords: "Generative AI, Google Cloud, Vertex AI, GCP, OAuth2, Credentials, Sandbox" --- @@ -200,3 +200,178 @@ The `google-cloud` provider type does not include any network policy endpoints by default. You must add endpoint rules to your sandbox policy for each GCP API the sandbox needs to reach. See "Using with GCP APIs" above for an example. + +## Vertex AI + +The `google-vertex-ai` provider gives selected sandboxes access to native +Google Vertex AI endpoints. OpenShell keeps refresh bootstrap material at the +gateway, rotates short-lived access tokens, and resolves token placeholders +only at endpoints authorized by the provider profile. + +OpenShell does not choose a model or transform a request. The workload uses the +native Vertex endpoint and request format for its selected model. + +### Prerequisites + +- A GCP project with the Vertex AI API enabled. +- A service account with the Vertex AI User role and a downloaded JSON key for + production, or gcloud Application Default Credentials for local development. +- Access to the selected model in the intended Vertex region. + +### Create a Vertex AI Provider + +Import the `google-vertex-ai` profile first: + +```shell +curl -LsSfO https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/google-vertex-ai.yaml +openshell provider profile import -f google-vertex-ai.yaml --global +``` + +#### Service Account Key + +Create the provider with the JSON key as gateway-only bootstrap material: + +```shell +openshell provider create \ + --name vertex-prod \ + --type google-vertex-ai \ + --credential GOOGLE_SERVICE_ACCOUNT_KEY="$(cat /path/to/key.json)" \ + --config VERTEX_AI_PROJECT_ID=my-gcp-project \ + --config VERTEX_AI_REGION=us-central1 +``` + +Configure gateway-managed refresh: + +```shell +openshell provider refresh configure vertex-prod \ + --credential-key GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN \ + --strategy google-service-account-jwt \ + --material client_email="sa@my-gcp-project.iam.gserviceaccount.com" \ + --material private_key="$(jq -r .private_key /path/to/key.json)" \ + --secret-material-key private_key +``` + +The private key remains in the gateway credential store. Sandboxes receive +only an opaque placeholder for the short-lived access token. + +#### gcloud Application Default Credentials + +For local development: + +```shell +gcloud auth application-default login + +openshell provider create \ + --name vertex-local \ + --type google-vertex-ai \ + --from-gcloud-adc \ + --config VERTEX_AI_PROJECT_ID=my-gcp-project \ + --config VERTEX_AI_REGION=us-central1 +``` + +`--from-gcloud-adc` reads authorized-user ADC, configures an OAuth2 refresh +grant at the gateway, and immediately mints `GOOGLE_VERTEX_AI_TOKEN`. The ADC +file and refresh token do not enter the sandbox. + +### Vertex AI Configuration Keys + +| Key | Required | Default | Description | +|---|---|---|---| +| `VERTEX_AI_PROJECT_ID` | Yes | — | GCP project ID exposed as non-secret workload configuration. | +| `VERTEX_AI_REGION` | No | `us-central1` | Vertex location exposed as non-secret workload configuration. | + +When the provider is attached, OpenShell also projects standard project and +location aliases such as `GOOGLE_CLOUD_PROJECT`, `ANTHROPIC_VERTEX_PROJECT_ID`, +`CLOUD_ML_REGION`, and `VERTEX_LOCATION`. + +### Attach the Vertex AI Provider + +Attach it while creating a sandbox: + +```shell +openshell sandbox create \ + --from registry.example.com/team/agent:1.0 \ + --name vertex-agent \ + --provider vertex-local +``` + +Or attach it to an existing sandbox: + +```shell +openshell sandbox provider attach vertex-agent vertex-local +``` + +Launch a new process after runtime attachment so it receives the provider +environment. Existing processes do not gain newly attached environment +variables. + +### Call the Native Vertex API + +Claude models use Vertex's publisher-model endpoint. Run a request from a new +sandbox process: + +```shell +openshell sandbox exec vertex-agent -- sh -lc ' + token=${GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN:-$GOOGLE_VERTEX_AI_TOKEN} + curl -X POST \ + -H "Authorization: Bearer $token" \ + -H "Content-Type: application/json" \ + -d '\''{ + "anthropic_version":"vertex-2023-10-16", + "max_tokens":1024, + "messages":[{"role":"user","content":"Hello"}] + }'\'' \ + "https://${CLOUD_ML_REGION}-aiplatform.googleapis.com/v1/projects/${GOOGLE_CLOUD_PROJECT}/locations/${CLOUD_ML_REGION}/publishers/anthropic/models/claude-sonnet-4-6:rawPredict" +' +``` + +Use the model ID and location supported by your GCP project. For `global`, `us`, +or `eu`, use the corresponding Google-documented hostname instead of the +regional `-aiplatform.googleapis.com` form. + +Gemini and third-party models use their documented native or +OpenAI-compatible Vertex endpoints. Configure the model, URL, streaming mode, +and timeout in the client. OpenShell does not rewrite them. + +### Verify and Troubleshoot Vertex AI + +Inspect the attachment and effective policy: + +```shell +openshell sandbox provider list vertex-agent +openshell policy get vertex-agent --full +openshell provider refresh status vertex-local +``` + +Common failures: + +- A missing token variable usually means the process started before provider + attachment. Launch a new process. +- `connection not allowed by policy` means the provider endpoint or caller + binary is absent from the effective policy. A gateway global policy override + suppresses provider-derived entries. +- `credential_endpoint_mismatch` means the request destination is outside the + provider profile's endpoint binding. +- A Vertex 400 or 404 usually means the model, location, publisher path, or + request body does not match the native API. +- A Vertex 401 or 403 can indicate an expired refresh grant or missing GCP IAM + permission. Check `provider refresh status` and the Vertex AI User role. + +Provider creation does not verify model access. The native request is the +end-to-end check. + +### Migrate an Existing Vertex Route + +An earlier managed route stored the provider and model separately and rewrote +requests for the workload. After upgrading, the provider and its refresh state +remain, but the route does not. + +1. Attach the preserved Vertex provider to each intended sandbox. +2. Launch new workload processes. +3. Move the route's model and timeout into the client configuration. +4. Change the client to the native Vertex endpoint and request format. +5. Verify one non-streaming and one streaming native request before production + rollout. + +Do not attach the provider to every sandbox automatically. The old route was +workspace-global; the replacement intentionally grants access per sandbox. diff --git a/docs/providers/google-vertex-ai.mdx b/docs/providers/google-vertex-ai.mdx deleted file mode 100644 index 0832baee90..0000000000 --- a/docs/providers/google-vertex-ai.mdx +++ /dev/null @@ -1,188 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Google Vertex AI" -sidebar-title: "Google Vertex AI" -description: "Attach a Google Vertex AI provider and call native Vertex endpoints with gateway-refreshed credentials." -keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Google Vertex AI, Anthropic Claude" ---- - -The `google-vertex-ai` provider gives selected sandboxes access to native -Google Vertex AI endpoints. OpenShell keeps refresh bootstrap material at the -gateway, rotates short-lived access tokens, and resolves token placeholders -only at endpoints authorized by the provider profile. - -OpenShell does not choose a model or transform a request. The workload uses the -native Vertex endpoint and request format for its selected model. - -## Prerequisites - -- A GCP project with the Vertex AI API enabled. -- A service account with the Vertex AI User role and a downloaded JSON key for - production, or gcloud Application Default Credentials for local development. -- Access to the selected model in the intended Vertex region. - -## Create a Provider - -Import the `google-vertex-ai` profile first; a gateway serves only the profiles you -imported: - -```shell -curl -LsSfO https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/google-vertex-ai.yaml -openshell provider profile import -f google-vertex-ai.yaml --global -``` - -### Service Account Key - -Create the provider with the JSON key as gateway-only bootstrap material: - -```shell -openshell provider create \ - --name vertex-prod \ - --type google-vertex-ai \ - --credential GOOGLE_SERVICE_ACCOUNT_KEY="$(cat /path/to/key.json)" \ - --config VERTEX_AI_PROJECT_ID=my-gcp-project \ - --config VERTEX_AI_REGION=us-central1 -``` - -Configure gateway-managed refresh: - -```shell -openshell provider refresh configure vertex-prod \ - --credential-key GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN \ - --strategy google-service-account-jwt \ - --material client_email="sa@my-gcp-project.iam.gserviceaccount.com" \ - --material private_key="$(jq -r .private_key /path/to/key.json)" \ - --secret-material-key private_key -``` - -The private key remains in the gateway credential store. Sandboxes receive -only an opaque placeholder for the short-lived access token. - -### gcloud Application Default Credentials - -For local development: - -```shell -gcloud auth application-default login - -openshell provider create \ - --name vertex-local \ - --type google-vertex-ai \ - --from-gcloud-adc \ - --config VERTEX_AI_PROJECT_ID=my-gcp-project \ - --config VERTEX_AI_REGION=us-central1 -``` - -`--from-gcloud-adc` reads authorized-user ADC, configures an OAuth2 refresh -grant at the gateway, and immediately mints `GOOGLE_VERTEX_AI_TOKEN`. The ADC -file and refresh token do not enter the sandbox. - -## Configuration Keys - -| Key | Required | Default | Description | -|---|---|---|---| -| `VERTEX_AI_PROJECT_ID` | Yes | — | GCP project ID exposed as non-secret workload configuration. | -| `VERTEX_AI_REGION` | No | `us-central1` | Vertex location exposed as non-secret workload configuration. | - -When the provider is attached, OpenShell also projects standard project and -location aliases such as `GOOGLE_CLOUD_PROJECT`, `ANTHROPIC_VERTEX_PROJECT_ID`, -`CLOUD_ML_REGION`, and `VERTEX_LOCATION`. - -## Attach the Provider - -Attach it while creating a sandbox: - -```shell -openshell sandbox create \ - --name vertex-agent \ - --provider vertex-local -``` - -Or attach it to an existing sandbox: - -```shell -openshell sandbox provider attach vertex-agent vertex-local -``` - -Launch a new process after runtime attachment so it receives the provider -environment. Existing processes do not gain newly attached environment -variables. - -## Call the Native Vertex API - -Claude models use Vertex's publisher-model endpoint. Run a request from a new -sandbox process: - -```shell -openshell sandbox exec vertex-agent -- sh -lc ' - token=${GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN:-$GOOGLE_VERTEX_AI_TOKEN} - curl -X POST \ - -H "Authorization: Bearer $token" \ - -H "Content-Type: application/json" \ - -d '\''{ - "anthropic_version":"vertex-2023-10-16", - "max_tokens":1024, - "messages":[{"role":"user","content":"Hello"}] - }'\'' \ - "https://${CLOUD_ML_REGION}-aiplatform.googleapis.com/v1/projects/${GOOGLE_CLOUD_PROJECT}/locations/${CLOUD_ML_REGION}/publishers/anthropic/models/claude-sonnet-4-6:rawPredict" -' -``` - -Use the model ID and location supported by your GCP project. For `global`, `us`, -or `eu`, use the corresponding Google-documented hostname instead of the -regional `-aiplatform.googleapis.com` form. - -Gemini and third-party models use their documented native or -OpenAI-compatible Vertex endpoints. Configure the model, URL, streaming mode, -and timeout in the client. OpenShell does not rewrite them. - -## Verify and Troubleshoot - -Inspect the attachment and effective policy: - -```shell -openshell sandbox provider list vertex-agent -openshell policy get vertex-agent --full -openshell provider refresh status vertex-local -``` - -Common failures: - -- A missing token variable usually means the process started before provider - attachment. Launch a new process. -- `connection not allowed by policy` means the provider endpoint or caller - binary is absent from the effective policy. A gateway global policy override - suppresses provider-derived entries. -- `credential_endpoint_mismatch` means the request destination is outside the - provider profile's endpoint binding. -- A Vertex 400 or 404 usually means the model, location, publisher path, or - request body does not match the native API. -- A Vertex 401 or 403 can indicate an expired refresh grant or missing GCP IAM - permission. Check `provider refresh status` and the Vertex AI User role. - -Provider creation does not verify model access. The native request is the -end-to-end check. - -## Migrate an Existing Vertex Route - -An earlier managed route stored the provider and model separately and rewrote -requests for the workload. After upgrading, the provider and its refresh state -remain, but the route does not. - -1. Attach the preserved Vertex provider to each intended sandbox. -2. Launch new workload processes. -3. Move the route's model and timeout into the client configuration. -4. Change the client to the native Vertex endpoint and request format. -5. Verify one non-streaming and one streaming native request before production - rollout. - -Do not attach the provider to every sandbox automatically. The old route was -workspace-global; the replacement intentionally grants access per sandbox. - -## Next Steps - -- [Provider-backed Inference](/sandboxes/inference-routing) -- [Profiles](/providers/profiles) -- [Providers](/sandboxes/manage-providers) -- [Customize Sandbox Policies](/sandboxes/policies) diff --git a/docs/providers/profiles.mdx b/docs/providers/profiles.mdx index 66f498ee11..446715a5c6 100644 --- a/docs/providers/profiles.mdx +++ b/docs/providers/profiles.mdx @@ -209,7 +209,7 @@ The following provider profile design items are not part of the current behavior | Policy prover integration | OpenShell does not yet run the policy prover automatically on sandbox startup or block startup based on prover findings. | | Refresh telemetry as OCSF events | Credential refresh logs are secret-safe gateway logs. OCSF refresh events and metrics are future work. | -Use [Provider-backed Inference](/sandboxes/inference-routing) to attach an +Use [Inference](/sandboxes/inference-routing) to attach an inference provider and call its native endpoint. ## Provider Profiles diff --git a/docs/about/container-gateway.mdx b/docs/reference/container-gateway.mdx similarity index 97% rename from docs/about/container-gateway.mdx rename to docs/reference/container-gateway.mdx index 57be7d0142..fc52cf2b1a 100644 --- a/docs/about/container-gateway.mdx +++ b/docs/reference/container-gateway.mdx @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Running the Gateway as a Container" -sidebar-title: "Container Gateway" +sidebar-title: "Container Deployment" description: "Run the OpenShell gateway using docker run or docker-compose without the installer." keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Docker, Podman, docker-compose, container, immutable OS, bootc, rpm-ostree" -position: 4 +position: 8 --- Use this approach when you want to run the OpenShell gateway as a container instead of installing it with the system package manager. This is useful on immutable OS distributions (Fedora CoreOS, bootc-based images, Silverblue) where the standard installer is not appropriate, or anywhere you prefer a container-first workflow. @@ -197,6 +197,6 @@ podman run -d \ ## Next Steps -- To create your first sandbox, refer to the [Quickstart](/get-started/quickstart). +- To prepare an image and launch an agent, refer to [Run Your First Agent](/about/run-an-agent). - To control what the agent can access, refer to [Policies](/sandboxes/policies). -- For environment variable reference, refer to [Sandbox Compute Drivers](/reference/sandbox-compute-drivers). +- For environment variable reference, refer to [Sandbox Runtimes](/reference/sandbox-compute-drivers). diff --git a/docs/reference/gateway-auth.mdx b/docs/reference/gateway-auth.mdx index 5dc8298c67..644060dd2b 100644 --- a/docs/reference/gateway-auth.mdx +++ b/docs/reference/gateway-auth.mdx @@ -199,7 +199,7 @@ For a headless environment, set `OPENSHELL_NO_BROWSER=1` before registering or l 6. The gateway authorizes the gRPC method. Platform-scoped methods require the configured admin role. Workspace-scoped methods require the configured user role and a sufficient membership in the target workspace. Admin role holders satisfy user-role checks and bypass workspace membership checks. For the Platform Admin, Workspace Admin, and Workspace User permissions, refer -to [Manage Workspaces and Access](/sandboxes/manage-workspaces). +to [Workspaces](/sandboxes/manage-workspaces). #### JWT validation @@ -242,7 +242,7 @@ display name when available, identity provider, roles, and scopes. The gateway returns its validated identity; the CLI does not infer these values from an unverified local token payload. Use the `subject` value when adding the user to a workspace. For membership commands, refer to -[Manage Workspaces and Access](/sandboxes/manage-workspaces). +[Workspaces](/sandboxes/manage-workspaces). ### Edge JWT (cloud gateways) diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 53bb8ceef0..ec4354aa1a 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Gateway Configuration File" -sidebar-title: "Gateway Config" +sidebar-title: "Configuration" description: "Reference for the OpenShell gateway TOML configuration file (RFC 0003)." keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Gateway, Configuration, TOML, Reference" position: 5 @@ -90,7 +90,7 @@ future version. To migrate an existing file: 5. Use canonical image pull policies: `always`, `if_not_present`, `never`, or Podman-only `newer`. Kubernetes-style capitalization and Podman's `missing` spelling are rejected. -6. Remove zero sentinels. Omit `gateway_jwt.ttl_secs` for a non-expiring token, +6. Remove zero sentinels. Omit `gateway_jwt.ttl_secs` to use the 900-second default, omit Docker or Podman `sandbox_pids_limit` to use OpenShell's default limit of 2048, and omit Podman `health_check_interval_secs` to disable health checks. Explicit zero values are invalid. @@ -197,7 +197,7 @@ signing_key_path = "/etc/openshell/jwt/signing.pem" public_key_path = "/etc/openshell/jwt/public.pem" kid_path = "/etc/openshell/jwt/kid" gateway_id = "openshell" -# Omit only for local single-player Docker, Podman, or VM gateways. +# Optional; defaults to 900 seconds. ttl_secs = 3600 [openshell.gateway.auth] @@ -257,7 +257,6 @@ sa_token_ttl_secs = 3600 [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" -allow_reference_namespace = false ``` Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to map a verified client certificate to a CLI user identity. This application-layer identity switch does not control the TLS handshake. When `client_ca_path` is set without OIDC, the listener requires a valid client certificate. When OIDC is configured, bearer-only clients may connect; the listener still validates any client certificate they present against the configured CA. Kubernetes deployments must leave `mtls_auth.enabled` unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. @@ -268,7 +267,7 @@ The client-certificate handshake policy is derived and has no `require_client_au `[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup keeps the workload unstarted until the effective policy and matching provider configuration pass admission. A rejected startup exposes `ConfigurationInvalid` and remains available for policy/provider repair in either mode. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. -`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. Omit it for a non-expiring token: the token has no `exp` claim and the response omits `expiration_time`. Use this only for local single-player Docker, Podman, or VM gateways. Explicit `0` is invalid. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway omits the field. +`[openshell.gateway.gateway_jwt] ttl_secs` controls the lifetime of gateway-minted, generation-bound sandbox session JWTs and typed extension JWTs. Omit it to use the 900-second default. Explicit `0` is invalid. Helm renders `3600` seconds by default. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. @@ -467,7 +466,7 @@ server: existingSecret: my-preprovisioned-kek-secret ``` -For `kubernetes-secrets`, `namespace` sets where OpenShell-managed provider Secret objects are stored. When omitted, the driver uses the in-cluster ServiceAccount namespace when available, otherwise `default`. The Helm chart creates a Role granting the gateway access to all Secrets in the credential namespace because OpenShell-managed Secret names are dynamic SHA-256 hashes that cannot be restricted with `resourceNames`. Deploy credential Secrets in a dedicated namespace (`server.credentialDrivers.kubernetesSecrets.namespace`) to limit the RBAC blast radius. +For `kubernetes-secrets`, `namespace` sets where OpenShell-managed provider Secret objects are stored in every workspace mode. When omitted, the driver uses the in-cluster ServiceAccount namespace when available, otherwise `default`. The driver rejects credential handles that reference any other namespace. The Helm chart creates a Role granting the gateway access to all Secrets in the credential namespace because OpenShell-managed Secret names are dynamic SHA-256 hashes that cannot be restricted with `resourceNames`; the gateway ClusterRole grants no credential Secret permissions. Deploy credential Secrets in a dedicated namespace (`server.credentialDrivers.kubernetesSecrets.namespace`) to limit the RBAC blast radius. When the credential namespace is the release namespace, the Role also covers the gateway's own TLS and database Secrets. Set `server.credentialDrivers.kubernetesSecrets.createNamespace=true` to have the chart create the dedicated namespace. The chart retains it on uninstall so stored credentials survive, and leaves an existing namespace it does not own untouched. For `vault`, `address` points at the Vault service. Non-loopback endpoints must use HTTPS; plaintext HTTP is accepted only for `localhost` or an IP loopback address during local development. Vault requests do not follow redirects, preventing credentials from being replayed to a downgraded or substituted endpoint. HTTPS uses platform trust roots by default. Set `ca_bundle` to a certificate-only PEM bundle for a private Vault CA; the bundle augments platform roots and normal hostname verification remains enabled. `mount` and `kv_version` describe the KV engine where OpenShell-managed provider secrets are stored, and `auth_method = "kubernetes"` logs in with the gateway Pod's ServiceAccount token. For local or development validation, use `auth_method = "token_file"` with `token_path = "/path/to/token"`. Do not put literal Vault tokens in TOML. @@ -588,10 +587,10 @@ authorization inputs. Image-pull Secrets are selected by gateway configuration, not by sandbox callers, and do not participate in resource admission. In managed mode OpenShell -copies each explicitly configured Secret into the workspace namespace with -gateway ownership metadata. An existing target Secret is updated only when its -ownership metadata matches the gateway and workspace. In shared and operator -modes, the configured Secret must already exist in the sandbox namespace. The +creates an immutable copy of each configured Secret for every sandbox runtime +generation and deletes it with that generation. Sandbox creation fails if a +Secret with the generation name already exists. In shared and operator modes, +the configured Secret must already exist in the sandbox namespace. The typed Kubernetes driver config does not support caller-selected Secret or ConfigMap volumes or environment references. Automatic service-account tokens and projected token volumes are prohibited. diff --git a/docs/reference/policy-prover.mdx b/docs/reference/policy-prover.mdx index f34106c83a..cc2bc9f97e 100644 --- a/docs/reference/policy-prover.mdx +++ b/docs/reference/policy-prover.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Standalone Policy Prover" -sidebar-title: "Policy Prover" +sidebar-title: "Prover" description: "Install and use the standalone OpenShell policy prover to check a local candidate policy against a managed boundary." keywords: "Generative AI, Cybersecurity, Policy, Prover, Boundary, Containment, CI" position: 4 diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index a72a21bb2c..b0c84cfa0c 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Policy Schema Reference" -sidebar-title: "Policy Schema" +sidebar-title: "Schema" description: "Complete field reference for the sandbox policy YAML including static and dynamic sections." keywords: "Generative AI, Cybersecurity, Policy, Schema, YAML, Reference, Security" position: 3 diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 2eb116902d..24511c8fae 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -1,9 +1,9 @@ --- # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Sandbox Compute Drivers" -sidebar-title: "Compute Drivers" -description: "Reference for Docker, Podman, MicroVM, Kubernetes, and Windows MXC sandbox compute drivers." +title: "Sandbox Runtimes" +sidebar-title: "Runtimes" +description: "Reference for Docker, Podman, MicroVM, Kubernetes, and Windows MXC sandbox runtimes." keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Docker, Podman, MicroVM, Kubernetes, MXC, Reference" position: 4 --- @@ -118,6 +118,7 @@ are driver names, such as `kubernetes`, and are not part of the nested schema. ```shell openshell sandbox create \ + --from registry.example.com/team/claude-code:1.0 \ --driver-config-json '{"kubernetes":{"pod":{"runtime_class_name":"kata-containers","priority_class_name":"batch-low"}}}' \ -- claude ``` @@ -226,6 +227,7 @@ docker volume create --label openshell.ai/sandbox-attachable=true \ --label openshell.ai/sandbox-attachable-workspace=default openshell-work openshell sandbox create \ + --from registry.example.com/team/claude-code:1.0 \ --driver-config-json '{"docker":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work","read_only":false}]}}' \ -- claude ``` @@ -251,6 +253,7 @@ enabled = false ```shell openshell sandbox create \ + --from registry.example.com/team/claude-code:1.0 \ --driver-config-json '{"docker":{"mounts":[{"type":"bind","source":"/srv/openshell/work","target":"/sandbox/work","read_only":false}]}}' \ -- claude ``` @@ -319,6 +322,7 @@ podman volume create --label openshell.ai/sandbox-attachable=true \ --label openshell.ai/sandbox-attachable-workspace=default openshell-work openshell sandbox create \ + --from registry.example.com/team/claude-code:1.0 \ --driver-config-json '{"podman":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work","read_only":false}]}}' \ -- claude ``` @@ -344,6 +348,7 @@ enabled = false ```shell openshell sandbox create \ + --from registry.example.com/team/claude-code:1.0 \ --driver-config-json '{"podman":{"mounts":[{"type":"bind","source":"/srv/openshell/work","target":"/sandbox/work","read_only":false}]}}' \ -- claude ``` @@ -397,7 +402,7 @@ compute_driver = "vm" For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. -Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The default sandbox image is `nvcr.io/nvidia/base/ubuntu:24.04`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. +Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The default workload and bootstrap image is `nvcr.io/nvidia/base/ubuntu:24.04`; configure a user-owned image with the agents and tools required by the workload. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. The gateway starts `openshell-driver-vm` over a private Unix socket and passes its process ID so the driver can reject unexpected local clients. The driver's standalone TCP listener is disabled unless `--allow-unauthenticated-tcp` is set for local development. @@ -456,16 +461,16 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | | `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | -| `default_image` | `server.sandboxImage` | Set the default sandbox image. | -| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | -| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | +| `default_image` | `sandbox.image.repository` / `sandbox.image.tag` / `sandbox.image.digest` | Set the default sandbox image. | +| `image_pull_policy` | `sandbox.image.pullPolicy` | Set the canonical sandbox pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | +| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode creates an immutable copy of each Secret from the configured source namespace for every sandbox runtime generation. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway endpoint reachable from sandbox pods. | -| `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | -| `sandbox_runtime_image` | `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` | Override the image that provides `openshell-sandbox`. The default repository with an empty tag uses the version pinned into the gateway. | +| `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Name the Kubernetes Secret holding sandbox client TLS materials. Shared mode mounts it directly; managed and operator modes stage its contents into each supervisor bootstrap Secret. | +| `sandbox_runtime_image` | `sandboxRuntime.image.registry` / `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` / `sandboxRuntime.image.digest` | Override the trusted workload-side image that provides `openshell-sandbox`. Individual image values take precedence over `global.image`; a digest takes precedence over the tag. | | `sandbox_runtime_image_pull_policy` | `sandboxRuntime.image.pullPolicy` | Set the Kubernetes image pull policy for the sandbox runtime image. | -| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the image that provides `openshell-supervisor`. The default repository with an empty tag uses the version pinned into the gateway. | -| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | +| `supervisor_image` | `supervisor.image.registry` / `supervisor.image.repository` / `supervisor.image.tag` / `supervisor.image.digest` | Override the image that provides `openshell-supervisor`. Individual image values take precedence over `global.image`; a digest takes precedence over the tag. | +| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | | `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` or `https://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | @@ -499,12 +504,12 @@ oc -n openshell create configmap corporate-proxy-ca --from-file=ca.crt=corp-ca.p Then set `upstreamProxy.caBundle.configMapName` to `corporate-proxy-ca` and leave `upstreamProxy.caBundle.key` at its `ca.crt` default. -Managed-mode Secret copying requires the gateway ServiceAccount to create -Secrets. Kubernetes RBAC cannot restrict Secret `create` by resource name, so -the Helm chart grants cluster-wide Secret `create`; Secret `get` and `patch` -remain limited to the explicitly configured TLS and image-pull Secret names. -The driver creates copies only in gateway-owned managed namespaces. Do not -reuse the gateway ServiceAccount for unrelated workloads. +Managed-mode sandbox runtime generations require the gateway ServiceAccount to +create and delete Secrets in workspace namespaces. Kubernetes RBAC cannot +restrict Secret `create` by resource name, so the Helm chart grants cluster-wide +Secret `create` and `delete`. Secret `get` is limited to the configured TLS and +image-pull source Secrets in the sandbox namespace. Do not reuse the gateway +ServiceAccount for unrelated workloads. The Kubernetes driver always places the sandbox runtime in the workload Pod and the supervisor in a separate, directly managed Pod. The @@ -542,6 +547,7 @@ OpenShell-created `/sandbox` workspace PVC. ```shell openshell sandbox create \ + --from registry.example.com/team/claude-code:1.0 \ --driver-config-json '{ "kubernetes": { "volumes": [{ diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index 575058e4b9..87e80104dd 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -47,11 +47,11 @@ OpenShell publishes standalone `openshell-prover` release assets for manual down These artifacts are attached to GitHub releases. The Linux binaries are static and do not require glibc. All prover archives include the required solver linkage. -## Compute Drivers +## Runtimes -The gateway can manage sandboxes through several compute drivers. +The gateway can manage sandboxes through several runtimes. -| Compute Driver | Status | Notes | +| Runtime | Status | Notes | |---|---|---| | Docker | Supported for local development and single-machine gateways. | Requires Docker Desktop or Docker Engine on the gateway host. | | Podman | Supported for rootless local and workstation workflows. | Requires a Podman-compatible socket and rootless networking setup. | @@ -91,8 +91,10 @@ To override the default image references, use Helm values: | Helm value | Purpose | |---|---| -| `image.repository` / `image.tag` | Override the gateway image reference. | -| `server.sandboxImage` | Override the default sandbox image. | +| `global.image.registry` / `global.image.tag` / `global.image.pullPolicy` | Override shared image settings for the gateway, supervisor, and trusted sandbox runtime. | +| `gateway.image.registry` / `gateway.image.repository` / `gateway.image.tag` / `gateway.image.digest` | Override the gateway image reference. | +| `supervisor.image.registry` / `supervisor.image.repository` / `supervisor.image.tag` / `supervisor.image.digest` | Override the supervisor image reference. | +| `sandbox.image.repository` / `sandbox.image.tag` / `sandbox.image.digest` | Override the default workload sandbox image. | ## Kernel Requirements @@ -141,6 +143,9 @@ qualification output as `seccomp_listener_mode` (`killable` or On macOS, these kernel modules run inside the Docker Desktop Linux VM, not on the host kernel. -## Agent Compatibility +## Agent Workloads -For the full list of supported agents and their default policy coverage, refer to the [Supported Agents](/about/supported-agents) page. +OpenShell runs agents and tools that you install in a user-owned OCI image. The +image must satisfy the selected compute driver's platform requirements. Refer to +[Run Your First Agent](/about/run-an-agent) for the image, provider, and policy +workflow. diff --git a/docs/sandboxes/inference-routing.mdx b/docs/sandboxes/inference-routing.mdx index b0234e9686..4ba63ea7ca 100644 --- a/docs/sandboxes/inference-routing.mdx +++ b/docs/sandboxes/inference-routing.mdx @@ -1,8 +1,8 @@ --- # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Provider-backed Inference" -sidebar-title: "Provider-backed Inference" +title: "Inference" +sidebar-title: "Inference" description: "Grant a sandbox access to model providers through provider profiles, attachments, and native endpoints." keywords: "Generative AI, Cybersecurity, Inference, Provider Profiles, Credentials, LLM" position: 8 @@ -60,6 +60,7 @@ openshell provider create \ openshell sandbox create \ --name inference-demo \ + --from registry.example.com/team/python-agent:1.0 \ --provider nvidia-prod \ -- python app.py ``` @@ -161,6 +162,7 @@ openshell provider create --name ollama --type ollama-openai openshell sandbox create \ --name ollama-client \ + --from registry.example.com/team/python-agent:1.0 \ --provider ollama \ --env OPENAI_BASE_URL=http://host.openshell.internal:11434/v1 \ -- python app.py @@ -295,7 +297,7 @@ Special cases require additional work: - Host-local services need `host.openshell.internal` or a reachable LAN/service hostname, not `127.0.0.1` or `localhost`. - Google Vertex AI clients must use the native Vertex endpoint and - authentication behavior. See [Google Vertex AI](/providers/google-vertex-ai). + authentication behavior. See [Google](/providers/google-cloud#vertex-ai). - A bridge-fronted AWS Bedrock deployment needs a custom profile that declares the bridge endpoint and allowed client binaries. @@ -313,4 +315,4 @@ headers, model selection, request shape, streaming, and timeout behavior. - [Profiles](/providers/profiles) - [Providers](/sandboxes/manage-providers) - [Customize Sandbox Policies](/sandboxes/policies) -- [Google Vertex AI](/providers/google-vertex-ai) +- [Google](/providers/google-cloud#vertex-ai) diff --git a/docs/sandboxes/manage-gateways.mdx b/docs/sandboxes/manage-gateways.mdx index 731fd52c12..0514e012ed 100644 --- a/docs/sandboxes/manage-gateways.mdx +++ b/docs/sandboxes/manage-gateways.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Manage Gateways" -sidebar-title: "Gateways" +sidebar-title: "Overview" description: "Register OpenShell gateways, switch between environments, inspect gateway status, and troubleshoot gateway access." keywords: "Generative AI, Cybersecurity, Gateway, Docker, Podman, Kubernetes, MicroVM, CLI" position: 2 @@ -20,11 +20,11 @@ The gateway is responsible for: OpenShell separates gateway access from the compute driver that runs sandboxes. Use [Installation](/about/installation) to install OpenShell, choose a compute driver, and start a gateway. This page covers working with gateway entries after a gateway exists. -## Gateway Compute Drivers +## Gateway Runtimes A gateway provisions sandboxes through the compute driver configured for that gateway. -| Compute Driver | Where sandboxes run | Best for | +| Runtime | Where sandboxes run | Best for | |---|---|---| | Docker | Containers on the gateway host. | Solo development, quick iteration, and single-machine gateways. | | Podman | Rootless containers on the gateway host. | Workstations that avoid a rootful Docker daemon. | @@ -195,5 +195,5 @@ For sandbox startup failures, inspect the selected compute driver: ## Next Steps - To install OpenShell and choose a compute driver, refer to [Installation](/about/installation). -- To configure workspace membership and roles, refer to [Manage Workspaces and Access](/sandboxes/manage-workspaces). +- To configure workspace membership and roles, refer to [Workspaces](/sandboxes/manage-workspaces). - To create a sandbox using the gateway, refer to [Manage Sandboxes](/sandboxes/manage-sandboxes). diff --git a/docs/sandboxes/manage-providers.mdx b/docs/sandboxes/manage-providers.mdx index a1837c0a07..8da46617e9 100644 --- a/docs/sandboxes/manage-providers.mdx +++ b/docs/sandboxes/manage-providers.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Providers" -sidebar-title: "Providers" +sidebar-title: "Overview" description: "Create and manage credential providers that inject API keys and tokens into OpenShell sandboxes." keywords: "Generative AI, Cybersecurity, Providers, Credentials, API Keys, Sandbox, Security" position: 4 @@ -476,14 +476,14 @@ An OpenAI-compatible protocol does not make the `openai` profile safe for an arbitrary host. Baseten, Bitdeer, Groq, Ollama, LM Studio, self-hosted NIM, and other alternate endpoints need their own profile declaring the actual host, port, credential, and allowed binaries. See -[Provider-backed Inference](/sandboxes/inference-routing) for complete examples +[Inference](/sandboxes/inference-routing) for complete examples and migration guidance. ## Next Steps Explore related topics: -- To manage workspace access for providers, refer to [Manage Workspaces and Access](/sandboxes/manage-workspaces). +- To manage workspace access for providers, refer to [Workspaces](/sandboxes/manage-workspaces). - To control what the agent can access, refer to [Policies](/sandboxes/policies). - To use the default workload image, refer to [Sandboxes](/sandboxes/manage-sandboxes#default-workload-image). - To view the complete field reference for the policy YAML, refer to the [Policy Schema Reference](/reference/policy-schema). diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index a5dc0880c0..a25eed30fc 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Manage Sandboxes" -sidebar-title: "Sandboxes" +sidebar-title: "Overview" description: "Create sandboxes, understand sandbox isolation, and manage the full sandbox lifecycle." keywords: "Generative AI, Cybersecurity, Gateway, Sandboxing, AI Agents, Sandbox Management, CLI" position: 1 @@ -297,7 +297,7 @@ Override it with any image visible to the active compute driver: openshell sandbox create --from registry.example.com/agents/my-agent:1.0 ``` -Refer to [Default Policy](/reference/default-policy), [Supported Agents](/about/supported-agents), and the [BYOC example](https://github.com/NVIDIA/OpenShell/tree/main/examples/bring-your-own-container). +Refer to [Default Policy](/reference/default-policy), [Run Your First Agent](/about/run-an-agent), and the [bring-your-own-container example](https://github.com/NVIDIA/OpenShell/tree/main/examples/bring-your-own-container). ## Connect to a Sandbox @@ -900,16 +900,16 @@ with reason `MainProcessCompleted`. Nonzero and signal-normalized results use signal exits use the standard `128 + signal` convention. Compute runtimes do not automatically restart that process. -## Sandbox Compute Drivers +## Sandbox Runtimes The gateway's configured compute driver determines how OpenShell creates each sandbox. The CLI workflow stays the same across drivers: you create, connect to, inspect, and delete sandboxes through the gateway API. -For Docker, Podman, MicroVM, and Kubernetes behavior, refer to [Sandbox Compute Drivers](/reference/sandbox-compute-drivers). +For Docker, Podman, MicroVM, and Kubernetes behavior, refer to [Sandbox Runtimes](/reference/sandbox-compute-drivers). ## Next Steps - To follow a complete end-to-end example, refer to the [GitHub Sandbox](/get-started/tutorials/github-sandbox) tutorial. -- To select a workspace or understand access roles, refer to [Manage Workspaces and Access](/sandboxes/manage-workspaces). +- To select a workspace or understand access roles, refer to [Workspaces](/sandboxes/manage-workspaces). - To supply API keys or tokens, refer to [Manage Providers](/sandboxes/manage-providers). - To control what the agent can access, refer to [Policies](/sandboxes/policies). - To use the default runtime image, refer to [Default Workload Image](#default-workload-image). diff --git a/docs/sandboxes/manage-workspaces.mdx b/docs/sandboxes/manage-workspaces.mdx index 7fb6a1ff24..74ae99a71d 100644 --- a/docs/sandboxes/manage-workspaces.mdx +++ b/docs/sandboxes/manage-workspaces.mdx @@ -1,8 +1,8 @@ --- # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Manage Workspaces and Access" -sidebar-title: "Workspaces and Access" +title: "Workspaces" +sidebar-title: "Workspaces" description: "Create OpenShell workspaces, assign members, and understand platform and workspace roles." keywords: "Generative AI, Cybersecurity, Workspaces, Access Control, RBAC, OIDC, Membership, CLI" position: 3 @@ -141,7 +141,11 @@ can appear before or after the subcommand. ```shell openshell sandbox list --workspace team-ml openshell provider list --workspace team-ml -openshell sandbox create --workspace team-ml --name research -- bash +openshell sandbox create \ + --workspace team-ml \ + --name research \ + --from registry.example.com/team/agent-tools:1.0 \ + -- bash ``` Set a default for the current shell with `OPENSHELL_WORKSPACE`: diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index d3fb6235d2..698701d505 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Customize Sandbox Policies" -sidebar-title: "Policies" +sidebar-title: "Overview" description: "Apply, iterate, and debug sandbox network policies with hot-reload on running OpenShell sandboxes." keywords: "Generative AI, Cybersecurity, Policy, Network Policy, Sandbox, Security, Hot Reload" position: 6 diff --git a/docs/sandboxes/policy-advisor.mdx b/docs/sandboxes/policy-advisor.mdx index 80884620d5..938bf93bd1 100644 --- a/docs/sandboxes/policy-advisor.mdx +++ b/docs/sandboxes/policy-advisor.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Use Policy Advisor" -sidebar-title: "Policy Advisor" +sidebar-title: "Advisor" description: "Let sandboxed agents propose narrow policy changes through policy.local while keeping developer approval in the loop." keywords: "Generative AI, Cybersecurity, Policy Advisor, Policy, Sandbox, policy.local, Agent Policy" position: 7 diff --git a/docs/sdk/go.mdx b/docs/sdk/go.mdx new file mode 100644 index 0000000000..478280934d --- /dev/null +++ b/docs/sdk/go.mdx @@ -0,0 +1,126 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Go SDK" +sidebar-title: "Go" +description: "Install the OpenShell Go SDK, connect to a gateway, and manage a sandbox." +keywords: "OpenShell, SDK, Go, Golang, gRPC, Sandbox" +position: 1 +--- + +Use the Go SDK to manage OpenShell resources from Go applications, operators, +and controllers. Its typed subclients follow familiar Kubernetes client +patterns. Use the SDK and gateway from the same OpenShell release when possible. + +## Install the SDK + +The SDK requires Go 1.25.13 or later. Add it to your Go module: + +```shell +go get github.com/NVIDIA/OpenShell/sdk/go@latest +``` + +## Connect to a Gateway + +Pass a `host:port` address to `NewClient`. For a local gateway that allows +unauthenticated plaintext connections: + +```go +package main + +import ( + "context" + "fmt" + "log" + "time" + + v1 "github.com/NVIDIA/OpenShell/sdk/go/openshell/v1" +) + +func main() { + client, err := v1.NewClient(v1.Config{ + Address: "127.0.0.1:8080", + Auth: v1.NoAuth(), + }) + if err != nil { + log.Fatal(err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + health, err := client.Health().Check(ctx) + if err != nil { + log.Fatal(err) + } + fmt.Printf("gateway healthy: %v\n", health.Healthy) +} +``` + +For an authenticated TLS gateway, provide the bearer token and TLS settings: + +```go +client, err := v1.NewClient(v1.Config{ + Address: "gateway.example.com:443", + Auth: v1.StaticToken(os.Getenv("OPENSHELL_TOKEN")), + TLS: &v1.TLSConfig{CAFile: "/path/to/ca.crt"}, +}) +``` + +Omit `CAFile` to use system roots. For renewable OIDC service credentials, use +`oidc.NewClientCredentialsAuth` from +`github.com/NVIDIA/OpenShell/sdk/go/openshell/v1/oidc`. + +## Create and Use a Sandbox + +Resource methods take an explicit workspace. The following example creates a +sandbox in `default`, waits for readiness, runs a command, and requests +deletion: + +```go +sandbox, err := client.Sandboxes().Create( + ctx, + "default", + "sdk-example", + &v1.SandboxSpec{ + Template: &v1.SandboxTemplate{ + Image: "registry.example.com/team/python-agent:1.0", + }, + }, + nil, +) +if err != nil { + log.Fatal(err) +} + +sandbox, err = client.Sandboxes().WaitReady(ctx, "default", sandbox.Name) +if err != nil { + log.Fatal(err) +} + +result, err := client.Exec().Run( + ctx, + "default", + sandbox.Name, + []string{"python", "-c", "print('hello from OpenShell')"}, +) +if err != nil { + log.Fatal(err) +} +fmt.Print(string(result.Stdout)) + +if _, err := client.Sandboxes().Delete(ctx, "default", sandbox.Name); err != nil { + log.Fatal(err) +} +``` + +The root client also exposes subclients for providers, services, files, SSH, +TCP forwarding, policy, configuration, templates, and workspaces. List methods +return lazy pagers so callers choose when to fetch the next page. + +## Next Steps + +- Review [Gateway Authentication](/reference/gateway-auth) before connecting a service to a production gateway. +- Review [API Errors](/reference/api-errors) for the gateway error model. +- Browse the [Go package reference](https://pkg.go.dev/github.com/NVIDIA/OpenShell/sdk/go/openshell/v1) for all exported types and methods. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx new file mode 100644 index 0000000000..c03478cdd5 --- /dev/null +++ b/docs/sdk/python.mdx @@ -0,0 +1,114 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Python SDK" +sidebar-title: "Python" +description: "Install the OpenShell Python SDK, reuse a registered gateway, and manage a sandbox." +keywords: "OpenShell, SDK, Python, PyPI, gRPC, Sandbox" +position: 3 +--- + +Use the Python SDK to manage sandboxes from applications, notebooks, and +automation. It can reuse gateway registration and authentication state created +by the OpenShell CLI. Use the SDK and gateway from the same OpenShell release +when possible. + +## Install the SDK + +The package requires Python 3.11 or later. Add it to your project with `uv`: + +```shell +uv add openshell +``` + +The `openshell` package contains the SDK. It does not install the OpenShell CLI. + +## Connect to a Registered Gateway + +Register and select a gateway with the CLI first. Then construct the client from +the active gateway: + +```python +from openshell import SandboxClient + +with SandboxClient.from_active_cluster() as client: + health = client.health() + print(health.version) +``` + +`from_active_cluster()` reads the selected gateway endpoint, TLS files, and +OIDC token state. It refreshes an expiring OIDC token by default and writes the +rotated bundle back for other OpenShell processes. + +For a direct local plaintext connection, pass a `host:port` endpoint: + +```python +from openshell import SandboxClient + +with SandboxClient("127.0.0.1:8080") as client: + print(client.health().version) +``` + +For service automation against an OIDC gateway, use +`ClientCredentialsAuth`. Non-loopback endpoints require TLS: + +```python +import os + +from openshell import ClientCredentialsAuth, SandboxClient + +auth = ClientCredentialsAuth( + issuer="https://idp.example.com/realms/openshell", + client_id="openshell-service", + client_secret=lambda: os.environ["OPENSHELL_OIDC_CLIENT_SECRET"], + audience="openshell-gateway", +) + +with SandboxClient.from_active_cluster(client_credentials=auth) as client: + print(client.health().version) +``` + +## Create and Use a Sandbox + +Python SDK methods take an explicit workspace. This example creates a sandbox, +waits for readiness, runs a command, and waits for deletion: + +```python +from openshell import SandboxClient + +with SandboxClient.from_active_cluster() as client: + sandbox = client.create( + workspace="default", + name="sdk-example", + ) + client.wait_ready( + sandbox.name, + workspace="default", + timeout_seconds=120, + ) + + result = client.exec( + sandbox.name, + ["python", "-c", "print('hello from OpenShell')"], + workspace="default", + ) + print(result.stdout, end="") + + deletion = client.delete(sandbox.name, workspace="default") + client.wait_deleted( + sandbox.name, + workspace="default", + expected_sandbox_id=deletion.sandbox_id, + ) +``` + +Use `create_session()` when you want an object that retains the sandbox name and +workspace for repeated exec, stop, start, and delete operations. List methods +return lazy `Pager` instances; use `list_all()` only when you want to fetch the +complete collection. + +## Next Steps + +- Review [Manage Sandboxes](/sandboxes/manage-sandboxes) for labels, templates, services, and Python SDK examples. +- Review [Gateway Authentication](/reference/gateway-auth) for OIDC and client credentials. +- Review [API Errors](/reference/api-errors) for structured error handling. diff --git a/docs/sdk/rust.mdx b/docs/sdk/rust.mdx new file mode 100644 index 0000000000..20e938a84c --- /dev/null +++ b/docs/sdk/rust.mdx @@ -0,0 +1,129 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Rust SDK" +sidebar-title: "Rust" +description: "Install the OpenShell Rust SDK, connect to a gateway, and manage a sandbox asynchronously." +keywords: "OpenShell, SDK, Rust, Cargo, Tonic, gRPC, Sandbox" +position: 2 +--- + +The Rust SDK is the asynchronous gateway client used by the OpenShell CLI and +TUI. Its curated API covers common sandbox operations, while `raw` exposes the +generated Tonic clients for the complete RPC surface. Use the SDK and gateway +from the same OpenShell release when possible. + +## Install the SDK + +The Rust SDK is currently consumed from source. Pin the dependency to the same +release tag as your gateway: + +```shell +cargo add openshell-sdk \ + --git https://github.com/NVIDIA/OpenShell \ + --tag +cargo add tokio --features macros,rt-multi-thread +``` + +## Connect to a Gateway + +`ClientConfig` accepts a full gateway URL. This local example uses plaintext +transport and no authentication: + +```rust +use openshell_sdk::{ClientConfig, OpenShellClient}; + +let client = OpenShellClient::connect( + ClientConfig::new("http://127.0.0.1:8080"), +) +.await?; + +let health = client.health().await?; +println!("gateway status: {:?}", health.status); +``` + +For an OIDC gateway, attach a bearer token and use HTTPS: + +```rust +use openshell_sdk::{AuthConfig, ClientConfig, OpenShellClient}; + +let client = OpenShellClient::connect(ClientConfig { + gateway: "https://gateway.example.com".to_string(), + auth: Some(AuthConfig::oidc(std::env::var("OPENSHELL_TOKEN")?)), + ..Default::default() +}) +.await?; +``` + +Set `ca_cert` when the gateway uses a private CA. The Rust SDK supports +server-authenticated TLS and OIDC bearer authentication. It does not support +mTLS client certificates or read the CLI gateway configuration from disk. + +## Create and Use a Sandbox + +Curated operations use the `default` workspace. Use +`client.workspace("name")` when your application targets another +workspace. + +```rust +use std::time::Duration; + +use openshell_sdk::{ + ClientConfig, DeleteOptions, ExecOptions, OpenShellClient, SandboxSpec, +}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let client = OpenShellClient::connect( + ClientConfig::new("http://127.0.0.1:8080"), + ) + .await?; + + let sandbox = client + .create_sandbox(SandboxSpec { + name: Some("sdk-example".to_string()), + image: Some( + "registry.example.com/team/python-agent:1.0" + .to_string(), + ), + ..Default::default() + }) + .await?; + + client + .wait_ready(&sandbox.name, Duration::from_secs(120)) + .await?; + + let result = client + .exec( + &sandbox.name, + &["python".into(), "-c".into(), "print('hello from OpenShell')".into()], + ExecOptions::default(), + ) + .await?; + print!("{}", String::from_utf8_lossy(&result.stdout)); + + let deletion = client + .delete_sandbox(&sandbox.name, DeleteOptions::default()) + .await?; + client + .wait_deleted( + &sandbox.name, + Duration::from_secs(60), + deletion.sandbox_id.as_deref(), + ) + .await?; + + Ok(()) +} +``` + +Use `raw_grpc_fresh()` for RPCs that the curated API does not yet wrap. The raw +client returns generated protobuf types and requires callers to assemble full +requests. + +## Next Steps + +- Review [Gateway Authentication](/reference/gateway-auth) for transport and identity choices. +- Review [API Errors](/reference/api-errors) for the gateway error model. +- See the [Rust SDK source documentation](https://github.com/NVIDIA/OpenShell/tree/main/crates/openshell-sdk) for modules and advanced transport behavior. diff --git a/docs/sdk/typescript.mdx b/docs/sdk/typescript.mdx new file mode 100644 index 0000000000..f9ac855480 --- /dev/null +++ b/docs/sdk/typescript.mdx @@ -0,0 +1,110 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "TypeScript SDK" +sidebar-title: "TypeScript" +description: "Install the OpenShell TypeScript SDK, connect to a gateway, and manage a sandbox." +keywords: "OpenShell, SDK, TypeScript, Node.js, Connect, gRPC, Sandbox" +position: 4 +--- + +Use the TypeScript SDK in Node.js applications and automation. It provides a +curated sandbox API and a raw generated client for the full gateway RPC surface. +Use the SDK and gateway from the same OpenShell release when possible. + +## Install the SDK + +The package requires Node.js 20.3 or later and is currently distributed through +GitHub Packages. Configure the `@nvidia` scope in your project `.npmrc`: + +```text +@nvidia:registry=https://npm.pkg.github.com +``` + +Authenticate npm with a GitHub token that has `read:packages`, then install the +package: + +```shell +npm install @nvidia/openshell-sdk +``` + +## Connect to a Gateway + +`OpenShellClient.connect()` accepts a gateway URL. It constructs a lazy client, +so call `health()` when startup must verify connectivity: + +```ts +import { OpenShellClient } from '@nvidia/openshell-sdk' + +const client = await OpenShellClient.connect({ + gateway: 'https://gateway.example.com', + oidcToken: process.env.OPENSHELL_TOKEN, +}) + +const health = await client.health() +console.log(`${health.status}: ${health.version}`) +``` + +For long-running OIDC service automation, use a renewable client-credentials +provider: + +```ts +import { clientCredentials, OpenShellClient } from '@nvidia/openshell-sdk' + +const client = await OpenShellClient.connect({ + gateway: 'https://gateway.example.com', + oidcTokenProvider: clientCredentials({ + issuer: 'https://idp.example.com/realms/openshell', + clientId: 'openshell-service', + clientSecret: () => process.env.OPENSHELL_OIDC_CLIENT_SECRET!, + audience: 'openshell-gateway', + }), +}) +``` + +The provider retains credentials and tokens in memory and renews the access +token before expiry. + +## Create and Use a Sandbox + +The curated client uses the `default` workspace unless you pass `workspace` in +an operation's options: + +```ts +const sandbox = await client.sandbox.create({ + name: 'sdk-example', + image: 'registry.example.com/team/python-agent:1.0', +}) + +await client.sandbox.waitReady(sandbox.name, 120) + +const result = await client.sandbox.exec( + sandbox.name, + ['python', '-c', "print('hello from OpenShell')"], +) +console.log(result.stdout.toString()) + +const deletion = await client.sandbox.delete(sandbox.name) +if (deletion.outcome === 'accepted') { + await client.sandbox.waitDeleted(sandbox.name, 60, { + expectedSandboxId: deletion.sandboxId, + }) +} +``` + +`client.sandbox` also supports streaming and interactive exec, TCP forwarding, +SSH sessions, sandbox provider attachment, configuration, and policy. Close +operation-scoped streams and forwarding handles when finished. The root client +does not retain a dedicated session and has no `close()` method. + +## Use the Raw Client + +Use `client.raw` for RPCs that the curated clients do not yet wrap. Import +generated message schemas and types from `@nvidia/openshell-sdk/raw`. Raw calls +return protobuf wire shapes, while curated calls return SDK-specific types. + +## Next Steps + +- Review [Gateway Authentication](/reference/gateway-auth) for OIDC and service authentication. +- Review [API Errors](/reference/api-errors) for structured error handling. +- See the [TypeScript SDK source documentation](https://github.com/NVIDIA/OpenShell/tree/main/sdk/typescript) for streaming, forwarding, and raw client examples. diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index a011677c23..40a1e18639 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -13,7 +13,7 @@ OpenShell enforces sandbox security across four layers: network, filesystem, pro This page documents every configurable control, its default, what it protects, and the risk of relaxing it. For the full policy YAML schema, refer to the [Policy Schema](/reference/policy-schema). -For the architecture of each enforcement layer, refer to [How OpenShell Works](/about/how-it-works). +For the architecture of each enforcement layer, refer to [Architecture](/about/how-it-works). If you use [NemoClaw](https://github.com/NVIDIA/NemoClaw), its [Security Best Practices](https://docs.nvidia.com/nemoclaw/latest/security/best-practices.html) guide covers additional entrypoint-level controls, policy presets, provider trust tiers, and posture profiles specific to the NemoClaw blueprint. @@ -79,16 +79,17 @@ This provides defense-in-depth: even if a container escape vulnerability exists, ### Binary Identity Binding -The proxy identifies which binary initiated each connection by reading `/proc//exe` (the kernel-trusted executable path). -It walks the process tree for ancestor binaries and parses `/proc//cmdline` for script interpreters. -The proxy SHA256-hashes each binary on first use (trust-on-first-use). If someone replaces a binary mid-session, the hash mismatch triggers an immediate deny. +OpenShell binds each connection to executable identity observed by the trusted isolation boundary. +Network policy can authorize either the connection-owning executable or one of its executable ancestors. +OpenShell pins every authorization-capable path to its first runtime-observed SHA256 digest (trust on first use), and a later digest mismatch triggers an immediate deny. +Missing or conflicting integrity evidence also fails closed. Command-line paths remain diagnostic context and never authorize access. | Aspect | Detail | |---|---| | Default | Every `network_policies` entry requires a `binaries` list. Only listed binaries can reach the associated endpoints. Binary paths support glob patterns (`*` for one path component, `**` for recursive). | | What you can change | Add binaries to an endpoint entry. Use glob patterns for directory-scoped access (for example, `/sandbox/.vscode-server/**`). | -| Risk if relaxed | Broad glob patterns (like `/**`) allow any binary to reach the endpoint, defeating the purpose of binary-scoped enforcement. | -| Recommendation | Scope binaries to the specific executables that need each endpoint. Use narrow globs when the exact path varies (for example, across Python virtual environments). | +| Risk if relaxed | Broad glob patterns (like `/**`) allow any leaf or ancestor executable to authorize the endpoint, defeating the purpose of binary-scoped enforcement. | +| Recommendation | Scope binaries to the specific leaf or launcher executables that need each endpoint. Use narrow globs when the exact path varies (for example, across Python virtual environments). Restart the sandbox runtime after intentionally replacing an executable at an authorized path so a new runtime can establish a new pin. | ### L4-Only vs L7 Inspection @@ -239,7 +240,7 @@ This ordering is intentional: named network-namespace setup still relies on priv 6. Landlock filesystem restrictions. 7. Runtime seccomp socket domain and syscall filters. -## Provider-backed Inference Controls +## Inference Controls Inference providers use the same endpoint-bound credential mechanism as other providers. Attach a profile-backed provider only to sandboxes that need it. @@ -299,5 +300,5 @@ The following patterns weaken security without providing meaningful benefit. - [Policy Schema](/reference/policy-schema) for the full field-by-field YAML reference. - [Default Policy](/reference/default-policy) for the built-in default policy breakdown. - [Gateway Auth](/reference/gateway-auth) for gateway authentication details. -- [How OpenShell Works](/about/how-it-works) for the system architecture. +- [Architecture](/about/how-it-works) for the system architecture. - NemoClaw [Security Best Practices](https://docs.nvidia.com/nemoclaw/latest/security/best-practices.html) for entrypoint-level controls (capability drops, PATH hardening, build toolchain removal), policy presets, provider trust tiers, and posture profiles. diff --git a/docs/upgrade/0-1-0.mdx b/docs/upgrade/0-1-0.mdx new file mode 100644 index 0000000000..f91322e3cd --- /dev/null +++ b/docs/upgrade/0-1-0.mdx @@ -0,0 +1,122 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Upgrade to NVIDIA OpenShell 0.1.0" +sidebar-title: "0.1.0" +description: "Prepare OpenShell operators and users for the breaking changes in version 0.1.0." +keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Upgrade, Migration" +--- + +Upgrading from OpenShell 0.0.x to 0.1.0? Start with the section that matches how you use the platform. + +Each item links to the pull request that defines the change. + +## Operators + +If you run OpenShell for a team, start here. + +- **Coordinate the upgrade.** Stop all gateway replicas, back up the database, and upgrade gateways, compute and credential drivers, supervisors, middleware, CLI clients, and SDK clients from the same release. The protocol, field-name, and time-type changes do not support mixed 0.0.x and 0.1.0 peers ([PR #3113](https://github.com/NVIDIA/OpenShell/pull/3113), [PR #3272](https://github.com/NVIDIA/OpenShell/pull/3272), [PR #3352](https://github.com/NVIDIA/OpenShell/pull/3352)). + +- **Recreate every sandbox.** Remove 0.0.x sandboxes before the upgrade and recreate them afterward. The persisted sandbox boundary and runtime descriptors are not compatible with 0.1.0 ([PR #2942](https://github.com/NVIDIA/OpenShell/pull/2942), [PR #3366](https://github.com/NVIDIA/OpenShell/pull/3366)). + +- **Export provider profiles before upgrading.** The gateway no longer includes built-in profiles. Import the saved profiles after the upgrade at the same global or workspace scope. Replace profile aliases such as `claude` and `gh` with the canonical IDs `claude-code` and `github` ([PR #2962](https://github.com/NVIDIA/OpenShell/pull/2962), [PR #3383](https://github.com/NVIDIA/OpenShell/pull/3383)). + + ```shell + openshell provider profile export -o yaml --global > .yaml + openshell profile import --from ./profiles --global + ``` + +- **Migrate `gateway.toml` to schema version 2.** Add the version, replace the plural compute-driver selector, move driver settings under `[openshell.drivers.]`, and apply the renamed Docker, Podman, and VM fields. Validate the file before restarting. See [Gateway Configuration](/reference/gateway-config#migrate-to-schema-version-2) for the complete field mapping and [PR #2814](https://github.com/NVIDIA/OpenShell/pull/2814) for the implementation. + + ```toml + [openshell] + version = 2 + + [openshell.gateway] + compute_driver = "kubernetes" + + [openshell.drivers.kubernetes] + namespace = "openshell" + ``` + + ```shell + openshell-gateway config preflight --path /etc/openshell/gateway.toml + ``` + +- **Move Helm application settings to `gatewayConfig`.** Express gateway settings with the schema-v2 TOML hierarchy. Keep TLS keys, passwords, client secrets, RBAC, Services, image settings, and volumes in the chart-owned values and Secret interfaces ([PR #3384](https://github.com/NVIDIA/OpenShell/pull/3384)). + +- **Review the default workload image.** The fallback becomes the minimal `nvcr.io/nvidia/base/ubuntu:24.04` image, which includes no agent CLI or image-baked policy. Bare image aliases are removed, so specify a fully qualified image that you built with the required agents, tools, and startup behavior. The runtime fallback policy denies network access and no longer grants `/app`; images without an OCI `USER` run as UID and GID `1000` ([PR #3386](https://github.com/NVIDIA/OpenShell/pull/3386)). + +- **Preserve credential encryption material.** New provider credentials use the active gateway credential driver, with encrypted database storage as the default. Preserve the generated key-encryption-key Secret, reconfigure refresh grants, and recreate credentials before switching credential drivers. Do not configure an empty `credential_drivers` list or run mixed gateway versions against the same refresh records ([PR #2437](https://github.com/NVIDIA/OpenShell/pull/2437)). + +- **Update Kubernetes user-namespace configuration.** Remove `platform_config.host_users`. Use the typed sandbox template `user_namespaces` field or the Kubernetes driver default ([PR #3248](https://github.com/NVIDIA/OpenShell/pull/3248)). + +- **Approve caller-selected compute resources.** Compute drivers now disable caller-supplied `driver_config` by default. If you enable it, label each allowed PVC, RuntimeClass, PriorityClass, Docker volume, or Podman volume with `openshell.ai/sandbox-attachable=true` and `openshell.ai/sandbox-attachable-workspace=`. Recreate legacy sandboxes without verifiable admission provenance, pass matching admission settings to standalone drivers, and set `gateway.allowDriverConfig=true` on each split Helm workspace release ([PR #3538](https://github.com/NVIDIA/OpenShell/pull/3538), [PR #3571](https://github.com/NVIDIA/OpenShell/pull/3571)). + +- **Review Helm PKI and ingress settings.** PKI initialization now fails an install or upgrade after its timeout by default. Increase `pkiInitJob.timeoutSeconds` when needed, and set `server.tls.enableMtls=false` when enabling `grpcRoute.backendTLSPolicy` ([PR #2728](https://github.com/NVIDIA/OpenShell/pull/2728)). + +- **Remove orphaned Helm RBAC objects.** After verifying the new namespace-qualified `ClusterRole` and `ClusterRoleBinding`, delete the old fixed-name `openshell-gateway-node-reader` objects if no release still owns them ([PR #2939](https://github.com/NVIDIA/OpenShell/pull/2939)). + +- **Recreate Kubernetes Secrets provider credentials.** The Kubernetes Secrets credential driver stores every provider credential in its configured `namespace` in every workspace mode. Remove the `workspace_mode`, `gateway_id`, and `allow_reference_namespace` driver settings; a `gateway.toml` that sets them is rejected at startup. Credentials stored by the driver are not migrated. Deploy a fresh gateway, then create providers and refresh grants again ([PR #3616](https://github.com/NVIDIA/OpenShell/pull/3616)). + +- **Install the workspace chart in operator-managed namespaces.** In operator workspace mode, the gateway receives Secret permissions only from the `openshell-workspace` chart. Install the matching chart release in every operator-managed namespace; without it, sandbox bootstrap fails ([PR #3616](https://github.com/NVIDIA/OpenShell/pull/3616)). + +- **Implement extension protocol negotiation.** Custom compute drivers, credential drivers, gateway interceptors, and middleware must exchange `PeerMetadata`, use protocol `1.0`, and advertise their family base capability. Upgrade both peers together. See [Extension Protocol Negotiation](/extensibility/extension-negotiation) and [PR #3352](https://github.com/NVIDIA/OpenShell/pull/3352). + +- **Remove compute-driver callback-listener negotiation.** Regenerate custom compute-driver bindings and connect supervisors to the operator-configured primary gateway endpoint ([PR #3365](https://github.com/NVIDIA/OpenShell/pull/3365)). + +- **Update supervisor middleware events.** Accept `MiddlewareDescribeRequest`, replace the WebSocket-specific terminal-event types with the shared middleware types, and handle the renamed and split end reasons ([PR #3073](https://github.com/NVIDIA/OpenShell/pull/3073)). + +- **Regenerate extension bindings.** Update canonical request field names and protobuf `Timestamp` and `Duration` fields. Do not reuse generated 0.0.x bindings with 0.1.0 peers ([PR #3113](https://github.com/NVIDIA/OpenShell/pull/3113), [PR #3272](https://github.com/NVIDIA/OpenShell/pull/3272)). + +- **Bind authenticated drivers to a runtime identity.** A custom compute driver that advertises `supports_sandbox_authentication` must return a non-empty `runtime_identity` from create, start, and authentication. It must also honor `expected_runtime_identity` during restart so the gateway can bind bootstrap credentials to the current compute resource ([PR #3531](https://github.com/NVIDIA/OpenShell/pull/3531)). + +## End users + +If you use OpenShell through the CLI, policies, APIs, or SDKs, review these changes. + +- **Build local images before sandbox creation.** `openshell sandbox create --from` no longer builds a Dockerfile or directory. Build and tag with the gateway's container engine, then pass the image reference. Remote gateways need an image they can pull from a registry ([PR #3214](https://github.com/NVIDIA/OpenShell/pull/3214)). + + ```shell + docker build -t registry.example.com/team/agent:0.1.0 . + docker push registry.example.com/team/agent:0.1.0 + openshell sandbox create --from registry.example.com/team/agent:0.1.0 + ``` + +- **Name providers explicitly.** A trailing sandbox command no longer infers or attaches a provider. Pass `--provider `, and ask the operator to import the referenced profile when it is missing ([PR #3383](https://github.com/NVIDIA/OpenShell/pull/3383)). + +- **Replace managed inference routes.** The `openshell inference` commands, route APIs, and `inference.local` endpoint are removed. Attach a provider to each sandbox and call its native endpoint with its native model and request format. See [Migrate from Managed Inference Routes](/sandboxes/inference-routing#migrate-from-managed-inference-routes) and [PR #3195](https://github.com/NVIDIA/OpenShell/pull/3195). + +- **Remove `NetworkBinary.harness`.** In authored policies, keep each binary as an object containing only `path`, such as `- path: /usr/bin/curl`. In provider profiles, write binaries as scalar paths such as `- /usr/bin/curl` ([PR #3222](https://github.com/NVIDIA/OpenShell/pull/3222)). + +- **Fix unknown policy fields.** The authored policy schema rejects misspelled, obsolete, and other unknown fields instead of ignoring them ([PR #3334](https://github.com/NVIDIA/OpenShell/pull/3334)). + +- **Update endpoint modes.** Regenerate clients for the typed `tls`, `enforcement`, and `access` enums. Remove `tls: terminate` and `tls: passthrough`; omit `tls` for automatic inspection. Do not use `skip` as a replacement because it disables inspection ([PR #3187](https://github.com/NVIDIA/OpenShell/pull/3187), [PR #3414](https://github.com/NVIDIA/OpenShell/pull/3414)). + +- **Target L7 policy edits explicitly.** Policy update commands that append allow or deny rules must include `--rule-name` and every `--binary`, or `--any-binary`. Raw API calls must send the complete `L7RuleTarget` ([PR #3380](https://github.com/NVIDIA/OpenShell/pull/3380)). + +- **Send the negotiated MCP version.** MCP clients must include one `MCP-Protocol-Version` header on each post-initialization request, and the endpoint policy must allow that revision ([PR #3241](https://github.com/NVIDIA/OpenShell/pull/3241)). + +- **Select workspaces explicitly.** Regenerate clients for `WorkspaceSelector`, and select the literal `default` workspace when appropriate. Omission no longer selects it. See [Manage Workspaces](/sandboxes/manage-workspaces) and [PR #3245](https://github.com/NVIDIA/OpenShell/pull/3245). + +- **Use canonical resource names.** Public sandbox RPCs accept a sandbox name plus its workspace instead of an internal sandbox ID. Update renamed request and JSON fields such as `sandbox`, `provider`, and `name` ([PR #3272](https://github.com/NVIDIA/OpenShell/pull/3272)). + +- **Use protobuf time types.** Replace scalar millisecond, second, and string fields with `google.protobuf.Timestamp` and `google.protobuf.Duration`. Preserve the distinction between an absent field and a zero value. See [Protobuf Time Types](/reference/protobuf-time-types) and [PR #3113](https://github.com/NVIDIA/OpenShell/pull/3113). + +- **Replace offset pagination.** Send `page_size` and the opaque `page_token`, then continue while `next_page_token` is non-empty. Curated SDK list methods may return lazy, single-pass pagers; use `list_all` or `ListAll` only when the full collection is required ([PR #3249](https://github.com/NVIDIA/OpenShell/pull/3249), [PR #3256](https://github.com/NVIDIA/OpenShell/pull/3256), [PR #3279](https://github.com/NVIDIA/OpenShell/pull/3279)). + +- **Handle typed deletion outcomes.** Replace `deleted`, `removed`, and `revoked` booleans with `DeletionOutcome`. Treat `ACCEPTED` as asynchronous, use the returned sandbox ID when waiting, and set `allow_missing` only when absence is acceptable. See [SDK Migration for Deletion](/reference/api-errors#sdk-migration-for-deletion) and [PR #3317](https://github.com/NVIDIA/OpenShell/pull/3317). + +- **Update SDK error handling.** Python clients raise `GatewayError`, which is a `grpc.RpcError` but not a `grpc.Call`. Rust error variants contain additional status fields. SDKs expose retry details but do not automatically retry mutations ([PR #3313](https://github.com/NVIDIA/OpenShell/pull/3313)). + +- **Use stable request IDs for retries.** When a mutation includes `request_id`, retry with the same identity, scope, method, and payload. Cancellation does not cancel admitted work. Reconcile the result after `REQUEST_OUTCOME_UNCERTAIN` instead of submitting a new ID. An admitted exec retry cannot replay output, its exit code, or the stream, so handle `REQUEST_STREAM_UNAVAILABLE` separately ([PR #3321](https://github.com/NVIDIA/OpenShell/pull/3321), [PR #3323](https://github.com/NVIDIA/OpenShell/pull/3323), [PR #3324](https://github.com/NVIDIA/OpenShell/pull/3324)). + +- **Drain interactive exec after closing input.** Closing the request stream closes stdin but does not end output. Drain output concurrently and wait for both the exit event and final RPC status. Go clients use `CloseInteractiveInput` or `CancelInteractive`; TypeScript clients use `closeInput()` or `cancel()` ([PR #3359](https://github.com/NVIDIA/OpenShell/pull/3359)). + +- **Resume watch streams by cursor.** Treat stream warnings as recoverable gaps, persist the greatest processed opaque cursor, and send it when reconnecting. If the gateway returns `OUT_OF_RANGE`, discard the cursor and restart without resume ([PR #3209](https://github.com/NVIDIA/OpenShell/pull/3209)). + +- **Stop importing persistence messages.** The `StoredProviderProfile`, refresh-state, stored policy revision, and draft-chunk messages move to a private storage package and have no public API replacement ([PR #3169](https://github.com/NVIDIA/OpenShell/pull/3169)). + +- **Replace scripted `gateway info` calls.** The command now returns live, admin-gated runtime state. Use `openshell gateway list -o json` for local registration metadata ([PR #2202](https://github.com/NVIDIA/OpenShell/pull/2202)). + +- **Stop parsing the profile list table.** The human-readable columns change. Use JSON or YAML output in scripts; the structured profile schema is unchanged ([PR #3258](https://github.com/NVIDIA/OpenShell/pull/3258)). diff --git a/e2e/mcp-conformance/README.md b/e2e/mcp-conformance/README.md index 8d244ab612..4d545281c6 100644 --- a/e2e/mcp-conformance/README.md +++ b/e2e/mcp-conformance/README.md @@ -38,8 +38,9 @@ on local Docker Desktop, or via `--add-host ...:host-gateway` on local Linux. The generated policy uses `protocol: mcp`, inserts the conformance runner's spec revision into the endpoint allowlist, and sets `mcp.allow_all_known_mcp_methods: true` so omitted rule methods use the endpoint MCP method profile. OpenShell enforces that allowlist on each non-initialize request using `MCP-Protocol-Version`, with `2025-03-26` as the missing-header fallback. The conformance runner selects the revision used by its client and server; OpenShell's request-version check does not yet provide complete revision-specific message parsing or response validation. The policy keeps OpenShell deny-by-default at the network boundary while allowing the upstream scenarios to exercise MCP behavior. The policy body lives in `policy-template.yaml`; the wrapper renders its MCP revision, host, port, and path placeholders from the upstream server URL. For local runs, the wrapper builds `openshell/supervisor:dev` automatically -when no supervisor image override is set. Set `OPENSHELL_DOCKER_SUPERVISOR_IMAGE` -or `OPENSHELL_SUPERVISOR_IMAGE` to use a prebuilt pullable image instead. +when no supervisor image override is set. Set `SUPERVISOR_IMAGE` to use a +prebuilt pullable image instead. The legacy `OPENSHELL_DOCKER_SUPERVISOR_IMAGE` +and `OPENSHELL_SUPERVISOR_IMAGE` overrides remain supported and take precedence. The pinned upstream checkout includes reference-client fixture drift that is tracked in `modelcontextprotocol/conformance#345`. The wrapper patches the diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 87da1c9396..d9721658a9 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -93,7 +93,7 @@ if ! [[ "${BASELINE_SHA}" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "ERROR: manifest baseline_commit must be a full 40-character SHA: ${MANIFEST}" >&2 exit 2 fi -BASELINE_SHA="${BASELINE_SHA,,}" +BASELINE_SHA="$(echo "$BASELINE_SHA" | tr '[:upper:]' '[:lower:]')" EXPECTED_CANDIDATE_SHA="$(git -C "${ROOT}" rev-parse HEAD)" if [ ! -d "${CANDIDATE_WORKTREE}" ]; then echo "ERROR: candidate worktree does not exist: ${CANDIDATE_WORKTREE}" >&2 diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 780f37df0f..5a7e95db7e 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -285,11 +285,19 @@ serve(fixture_listener) Host(HostPythonFixture), Network(SupportContainer), } - let (fixture, policy_host, real_ip, fixture_port, tcp_dns_port, transparent_port) = + let (fixture, policy_host, real_ip, fixture_port, tcp_dns_port, transparent_port, wrong_port) = if is_e2e_driver("docker") { let fixture_host_port = find_free_port(); let tcp_dns_host_port = find_free_port(); let transparent_host_port = find_free_port(); + let wrong_host_port = loop { + let candidate = find_free_port(); + if ![fixture_host_port, tcp_dns_host_port, transparent_host_port] + .contains(&candidate) + { + break candidate; + } + }; let fixture = HostPythonFixture::start( &fixture_script(fixture_host_port, tcp_dns_host_port, transparent_host_port), fixture_host_port, @@ -303,6 +311,7 @@ serve(fixture_listener) fixture_host_port, tcp_dns_host_port, transparent_host_port, + wrong_host_port, ) } else { let fixture = SupportContainer::start_python_with_capabilities( @@ -321,6 +330,7 @@ serve(fixture_listener) FIXTURE_PORT, TCP_DNS_PORT, TRANSPARENT_LISTENER_PORT, + FIXTURE_PORT + 1, ) }; // Keep the fixture alive through the sandbox assertions. @@ -373,7 +383,7 @@ print('transparent-tcp-e2e-ok') host = policy_host, port = fixture_port, tcp_dns_port = tcp_dns_port, - wrong_port = fixture_port + 1, + wrong_port = wrong_port, real_ip = real_ip, transparent_port = transparent_port, ); diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index eae13e2cf6..a8f97cf5ef 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -180,21 +180,27 @@ async fn managed_creates_namespace_with_labels() { let (ok, _) = kubectl(&["get", "serviceaccount", "openshell-sandbox", "-n", &ns]).await; assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); - // The managed driver copies only explicitly configured image-pull Secrets - // from the gateway namespace into the workspace namespace. - let (ok, copied_secret) = kubectl(&[ + // Each runtime generation gets its own immutable copy of the configured + // image-pull Secret; the configured name is never created in {ns}. + let (ok, staged) = kubectl(&[ "get", "secret", - "e2e-regcred", "-n", &ns, + "-l", + "openshell.ai/component=image-pull", "-o", - "jsonpath={.type}", + "jsonpath={range .items[*]}{.metadata.name} {.type} {.immutable}{\"\\n\"}{end}", ]) .await; assert!( - ok && copied_secret.contains("kubernetes.io/dockerconfigjson"), - "configured image-pull Secret should be copied into {ns}: {copied_secret}" + ok && staged.contains("os-pull-") && staged.contains("kubernetes.io/dockerconfigjson true"), + "a generation image-pull Secret should exist in {ns}: {staged}" + ); + let (ok, _) = kubectl(&["get", "secret", "e2e-regcred", "-n", &ns]).await; + assert!( + !ok, + "configured image-pull Secret name must not exist in {ns}" ); // Verify SSH ingress is restricted to the gateway peer. Because Kubernetes @@ -484,7 +490,7 @@ async fn managed_workspace_delete_removes_namespace() { } #[tokio::test] -async fn managed_tls_secret_copied_to_namespace() { +async fn managed_supervisor_reads_client_tls_from_its_bootstrap_secret() { let (ok, config_out) = kubectl(&[ "get", "configmap", @@ -496,7 +502,7 @@ async fn managed_tls_secret_copied_to_namespace() { ]) .await; if !ok || !config_out.contains("client_tls_secret_name") { - eprintln!("SKIP: client_tls_secret_name not configured; TLS secret copying disabled"); + eprintln!("SKIP: client_tls_secret_name not configured"); return; } @@ -528,26 +534,77 @@ async fn managed_tls_secret_copied_to_namespace() { "sandbox output missing expected string: {out}" ); - let (ok, out) = kubectl(&["get", "secret", "openshell-client-tls", "-n", &ns]).await; + let (ok, _) = kubectl(&["get", "secret", "openshell-client-tls", "-n", &ns]).await; assert!( - ok, - "TLS secret openshell-client-tls should be copied to managed namespace {ns}: {out}" + !ok, + "client TLS Secret must not be copied into managed namespace {ns}" ); - let (ok, label_out) = kubectl(&[ + let (ok, keys) = kubectl(&[ "get", "secret", - "openshell-client-tls", "-n", &ns, + "-l", + "openshell.ai/component=supervisor-bootstrap", "-o", - "jsonpath={.metadata.labels}", + "jsonpath={.items[*].data}", ]) .await; - assert!(ok, "failed to read TLS secret labels: {label_out}"); assert!( - label_out.contains("openshell.ai/managed-by"), - "copied TLS secret missing managed-by label: {label_out}" + ok && keys.contains("client-tls.crt") && keys.contains("client-ca.crt"), + "supervisor bootstrap Secret should carry client TLS material: {keys}" + ); +} + +const GATEWAY_SERVICE_ACCOUNT: &str = "system:serviceaccount:openshell:openshell"; + +async fn gateway_can(verb: &str, resource: &str, namespace: &str) -> bool { + let (_, out) = kubectl(&[ + "auth", + "can-i", + verb, + resource, + "-n", + namespace, + "--as", + GATEWAY_SERVICE_ACCOUNT, + ]) + .await; + match out.trim() { + "yes" => true, + "no" => false, + other => panic!("unexpected kubectl auth can-i output for {verb} {resource}: {other}"), + } +} + +#[tokio::test] +async fn managed_gateway_cannot_read_or_modify_workspace_secrets() { + let ns = managed_namespace(&unique_workspace("mgdrbac")); + + for verb in ["get", "patch"] { + assert!( + !gateway_can(verb, "secrets", &ns).await, + "gateway must not {verb} arbitrary Secrets in workspace namespace {ns}" + ); + assert!( + !gateway_can(verb, "secrets/unrelated-secret", &ns).await, + "gateway must not {verb} an unrelated Secret in workspace namespace {ns}" + ); + } + assert!( + !gateway_can("list", "secrets", &ns).await, + "gateway must not list Secrets in workspace namespace {ns}" + ); + for copied in ["secrets/openshell-client-tls", "secrets/e2e-regcred"] { + assert!( + !gateway_can("get", copied, &ns).await, + "gateway must not read {copied} outside its sandbox namespace" + ); + } + assert!( + gateway_can("get", "secrets", "openshell").await, + "gateway must still reach provider credential Secrets in its configured namespace" ); } diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs index a874c1188b..926afa800c 100644 --- a/e2e/rust/tests/workspace_namespace_operator.rs +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -88,8 +88,37 @@ async fn provision_operator_namespace(name: &str) { let (ok, out) = kubectl(&["label", "namespace", name, OPERATOR_LABEL]).await; assert!(ok, "failed to label namespace {name}: {out}"); - let (ok, out) = kubectl(&["create", "serviceaccount", SA_NAME, "-n", name]).await; - assert!(ok, "failed to create SA in {name}: {out}"); + let (ok, out) = install_workspace_chart(name).await; + assert!(ok, "failed to install workspace chart in {name}: {out}"); +} + +async fn install_workspace_chart(namespace: &str) -> (bool, String) { + let chart = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../deploy/helm/openshell-workspace" + ); + let output = tokio::process::Command::new("helm") + .args([ + "--kube-context", + &kube_context(), + "install", + "openshell-workspace", + chart, + "--namespace", + namespace, + "--wait", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("failed to spawn helm"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), combined) } async fn delete_namespace(name: &str) { @@ -143,6 +172,43 @@ impl Drop for OperatorCleanup { } } +const GATEWAY_SERVICE_ACCOUNT: &str = "system:serviceaccount:openshell:openshell"; + +async fn gateway_can(verb: &str, resource: &str, namespace: &str) -> bool { + let (_, out) = kubectl(&[ + "auth", + "can-i", + verb, + resource, + "-n", + namespace, + "--as", + GATEWAY_SERVICE_ACCOUNT, + ]) + .await; + match out.trim() { + "yes" => true, + "no" => false, + other => panic!("unexpected kubectl auth can-i output for {verb} {resource}: {other}"), + } +} + +#[tokio::test] +async fn operator_gateway_has_no_secret_access_without_workspace_chart() { + let ns = unique_namespace("oprbac"); + + for verb in ["get", "list", "create", "patch", "delete"] { + assert!( + !gateway_can(verb, "secrets", &ns).await, + "gateway must not {verb} Secrets in namespace {ns} without the workspace chart" + ); + } + assert!( + !gateway_can("patch", "secrets/openshell-client-tls", &ns).await, + "gateway must not patch the client TLS Secret in namespace {ns} without the workspace chart" + ); +} + #[tokio::test] async fn operator_sandbox_in_labeled_namespace() { let ns = unique_namespace("op"); diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index f9079c7bfd..d13d258479 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -9,6 +9,88 @@ # Keep an explicit override so telemetry-specific tests can opt back in. export OPENSHELL_TELEMETRY_ENABLED="${OPENSHELL_TELEMETRY_ENABLED:-false}" +# Resolve a test image override. Repository-only values inherit the caller's +# tag, while tagged and digest-pinned references are already complete. +e2e_image_reference_is_complete() { + local image=$1 + local last_component="${image##*/}" + + [[ "${image}" == *@* || "${last_component}" == *:* ]] +} + +e2e_image_reference_has_digest() { + [[ "$1" == *@* ]] +} + +e2e_resolve_image_reference() { + local image=$1 + local tag=$2 + + if e2e_image_reference_is_complete "${image}"; then + printf '%s\n' "${image}" + else + printf '%s:%s\n' "${image%/}" "${tag}" + fi +} + +e2e_image_reference_repository() { + local image=$1 + local repository="${image%%@*}" + local last_component="${repository##*/}" + + if [[ "${last_component}" == *:* ]]; then + repository="${repository%:*}" + fi + printf '%s\n' "${repository}" +} + +# Return the registry portion of an image repository. Docker treats the first +# path component as a registry when it contains a dot or colon, or is +# `localhost`; otherwise the image uses the configured/default registry. +e2e_image_reference_registry() { + local repository + local first_component + + repository="$(e2e_image_reference_repository "$1")" + first_component="${repository%%/*}" + if [[ "${repository}" == */* ]] \ + && { [[ "${first_component}" == *.* ]] || [[ "${first_component}" == *:* ]] || [[ "${first_component}" == "localhost" ]]; }; then + printf '%s\n' "${first_component}" + fi +} + +# Return the repository path without its registry, suitable for Helm's +# .image.repository values. +e2e_image_reference_repository_path() { + local repository + local registry + + repository="$(e2e_image_reference_repository "$1")" + registry="$(e2e_image_reference_registry "$1")" + if [ -n "${registry}" ]; then + printf '%s\n' "${repository#"${registry}"/}" + else + printf '%s\n' "${repository}" + fi +} + +e2e_image_reference_tag() { + local image=$1 + local repository="${image%%@*}" + local last_component="${repository##*/}" + + if [[ "${image}" == *@* || "${last_component}" != *:* ]]; then + return 0 + fi + printf '%s\n' "${last_component##*:}" +} + +e2e_image_reference_digest() { + if [[ "$1" == *@* ]]; then + printf '%s\n' "${1#*@}" + fi +} + e2e_cargo_target_dir() { local root=$1 shift diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index c4b5a446ed..9d1197a21d 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -17,6 +17,11 @@ # Sandbox image overrides: # OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE=... # OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE_PULL_POLICY=always|if_not_present|never +# SANDBOX_IMAGE=... (trusted sandbox runtime override) +# Supervisor image overrides: +# SUPERVISOR_IMAGE=... (common test-wrapper override) +# OPENSHELL_SUPERVISOR_IMAGE=... (existing compatibility override) +# OPENSHELL_DOCKER_SUPERVISOR_IMAGE=... (Docker-specific override) # # The default sandbox image uses a mutable tag. This wrapper refreshes it # before starting the gateway, while the Docker driver defaults to @@ -323,6 +328,16 @@ resolve_docker_supervisor_image() { return 0 fi + if [ -n "${SUPERVISOR_IMAGE:-}" ]; then + if [ -n "${CI:-}" ] && [ -z "${IMAGE_TAG:-}" ] \ + && ! e2e_image_reference_is_complete "${SUPERVISOR_IMAGE}"; then + echo "ERROR: IMAGE_TAG must be set in CI when SUPERVISOR_IMAGE is repository-only." >&2 + exit 2 + fi + printf '%s\n' "$(e2e_resolve_image_reference "${SUPERVISOR_IMAGE}" "${IMAGE_TAG:-dev}")" + return 0 + fi + if [ -n "${CI:-}" ]; then if [ -z "${IMAGE_TAG:-}" ]; then echo "ERROR: IMAGE_TAG must be set in CI when no Docker supervisor image override is provided." >&2 @@ -347,6 +362,10 @@ resolve_docker_sandbox_runtime_image() { printf '%s\n' "${OPENSHELL_SANDBOX_RUNTIME_IMAGE}" return 0 fi + if [ -n "${SANDBOX_IMAGE:-}" ]; then + printf '%s\n' "$(e2e_resolve_image_reference "${SANDBOX_IMAGE}" "${IMAGE_TAG:-dev}")" + return 0 + fi if [ -n "${CI:-}" ]; then if [ -z "${IMAGE_TAG:-}" ]; then @@ -443,7 +462,7 @@ ensure_docker_supervisor_image() { fi echo "ERROR: supervisor image '${image}' is not available." >&2 - echo " Build it, push it, or set OPENSHELL_SUPERVISOR_IMAGE to a pullable image." >&2 + echo " Build it, push it, or set SUPERVISOR_IMAGE/OPENSHELL_SUPERVISOR_IMAGE to a pullable image." >&2 exit 2 } diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index cb9a0f8820..5d00e11adb 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -652,12 +652,9 @@ run_scenario() { --namespace "${NAMESPACE}" --create-namespace \ "${helm_values_args[@]}" \ --set "fullnameOverride=openshell" \ - --set "image.repository=${REGISTRY_VALUE}/gateway" \ - --set "image.tag=${IMAGE_TAG_VALUE}" \ - --set "sandboxRuntime.image.repository=${REGISTRY_VALUE}/sandbox" \ - --set "sandboxRuntime.image.tag=${IMAGE_TAG_VALUE}" \ - --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ - --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ + "${GATEWAY_HELM_IMAGE_ARGS[@]}" \ + "${SUPERVISOR_HELM_IMAGE_ARGS[@]}" \ + "${SANDBOX_RUNTIME_HELM_IMAGE_ARGS[@]}" \ "${helm_post_renderer_args[@]}" \ "$@" \ --wait --timeout 5m @@ -940,6 +937,14 @@ else IMAGE_TAG_VALUE="${IMAGE_TAG:-latest}" fi REGISTRY_VALUE="${REGISTRY_VALUE%/}" +GATEWAY_IMAGE="$(e2e_resolve_image_reference "${GATEWAY_IMAGE:-${REGISTRY_VALUE}/gateway}" "${IMAGE_TAG_VALUE}")" +SUPERVISOR_IMAGE="$(e2e_resolve_image_reference "${SUPERVISOR_IMAGE:-${REGISTRY_VALUE}/supervisor}" "${IMAGE_TAG_VALUE}")" +SANDBOX_RUNTIME_IMAGE="$(e2e_resolve_image_reference "${SANDBOX_IMAGE:-${REGISTRY_VALUE}/sandbox}" "${IMAGE_TAG_VALUE}")" +BUILD_GATEWAY_IMAGE="${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" +BUILD_SUPERVISOR_IMAGE="${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" +GATEWAY_HELM_IMAGE_ARGS=(--set-string "gateway.image.registry=$(e2e_image_reference_registry "${GATEWAY_IMAGE}")" --set-string "gateway.image.repository=$(e2e_image_reference_repository_path "${GATEWAY_IMAGE}")" --set-string "gateway.image.tag=$(e2e_image_reference_tag "${GATEWAY_IMAGE}")" --set-string "gateway.image.digest=$(e2e_image_reference_digest "${GATEWAY_IMAGE}")") +SUPERVISOR_HELM_IMAGE_ARGS=(--set-string "supervisor.image.registry=$(e2e_image_reference_registry "${SUPERVISOR_IMAGE}")" --set-string "supervisor.image.repository=$(e2e_image_reference_repository_path "${SUPERVISOR_IMAGE}")" --set-string "supervisor.image.tag=$(e2e_image_reference_tag "${SUPERVISOR_IMAGE}")" --set-string "supervisor.image.digest=$(e2e_image_reference_digest "${SUPERVISOR_IMAGE}")") +SANDBOX_RUNTIME_HELM_IMAGE_ARGS=(--set-string "sandboxRuntime.image.registry=$(e2e_image_reference_registry "${SANDBOX_RUNTIME_IMAGE}")" --set-string "sandboxRuntime.image.repository=$(e2e_image_reference_repository_path "${SANDBOX_RUNTIME_IMAGE}")" --set-string "sandboxRuntime.image.tag=$(e2e_image_reference_tag "${SANDBOX_RUNTIME_IMAGE}")" --set-string "sandboxRuntime.image.digest=$(e2e_image_reference_digest "${SANDBOX_RUNTIME_IMAGE}")") # Resolve a host-gateway IP that sandbox pods can dial to reach test fixtures # running on the developer/CI host (HTTP fixtures bound to 0.0.0.0 plus sibling @@ -1032,7 +1037,7 @@ elif [[ "${KUBE_CONTEXT}" == k3d-* ]] && command -v k3d >/dev/null 2>&1; then fi if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then require_cmd docker - echo "Building local Kubernetes e2e images (${REGISTRY_VALUE}/{gateway,sandbox,supervisor}:${IMAGE_TAG_VALUE})..." + echo "Building local Kubernetes e2e images (${BUILD_GATEWAY_IMAGE}, ${BUILD_SUPERVISOR_IMAGE})..." if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then if [ "$(uname -s)" != "Linux" ]; then echo "ERROR: external Kubernetes driver image composition currently requires a Linux build host." >&2 @@ -1061,9 +1066,9 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then cp "${external_driver}" "${external_stage}/openshell-driver-kubernetes" docker build \ --build-arg "TARGETARCH=${external_arch}" \ - --build-arg "SUPERVISOR_IMAGE=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" \ + --build-arg "SUPERVISOR_IMAGE=${BUILD_SUPERVISOR_IMAGE}" \ --build-arg "SANDBOX_RUNTIME_IMAGE=${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" \ - --tag "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ + --tag "${BUILD_GATEWAY_IMAGE}" \ --file "${ROOT}/e2e/docker/Dockerfile.external-kubernetes-gateway" \ "${ROOT}" else @@ -1071,7 +1076,11 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then bash "${ROOT}/tasks/scripts/docker-build-image.sh" gateway fi sandbox_image="${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" - supervisor_image="${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" + if [ "${GATEWAY_IMAGE}" != "${BUILD_GATEWAY_IMAGE}" ]; then + if e2e_image_reference_has_digest "${GATEWAY_IMAGE}"; then echo "ERROR: digest-pinned GATEWAY_IMAGE requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=0" >&2; exit 2; fi + docker tag "${BUILD_GATEWAY_IMAGE}" "${GATEWAY_IMAGE}" + fi + supervisor_image="${BUILD_SUPERVISOR_IMAGE}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" != "1" ] \ || ! docker image inspect "${sandbox_image}" >/dev/null 2>&1; then CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ @@ -1080,6 +1089,13 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then reuse_sandbox_image=1 echo "Reusing existing sandbox image ${sandbox_image}" fi + if [ "${SANDBOX_RUNTIME_IMAGE}" != "${sandbox_image}" ]; then + if e2e_image_reference_has_digest "${SANDBOX_RUNTIME_IMAGE}"; then + echo "ERROR: digest-pinned SANDBOX_IMAGE requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=0" >&2 + exit 2 + fi + docker tag "${sandbox_image}" "${SANDBOX_RUNTIME_IMAGE}" + fi if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" != "1" ] \ || ! docker image inspect "${supervisor_image}" >/dev/null 2>&1; then CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ @@ -1088,13 +1104,18 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then reuse_supervisor_image=1 echo "Reusing existing supervisor image ${supervisor_image}" fi + if [ "${SUPERVISOR_IMAGE}" != "${BUILD_SUPERVISOR_IMAGE}" ]; then + if e2e_image_reference_has_digest "${SUPERVISOR_IMAGE}"; then echo "ERROR: digest-pinned SUPERVISOR_IMAGE requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=0" >&2; exit 2; fi + docker tag "${BUILD_SUPERVISOR_IMAGE}" "${SUPERVISOR_IMAGE}" + fi fi if [ -n "${import_cluster_name}" ]; then for image in \ - "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ + "${GATEWAY_IMAGE}" \ "${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" \ - "${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}"; do + "${SUPERVISOR_IMAGE}" \ + "${SANDBOX_RUNTIME_IMAGE}"; do if docker image inspect "${image}" >/dev/null 2>&1; then echo "Importing ${image} into k3d cluster ${import_cluster_name}..." k3d image import "${image}" --cluster "${import_cluster_name}" \ @@ -1105,16 +1126,18 @@ elif [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ] \ && [[ "${KUBE_CONTEXT}" == kind-* ]] \ && command -v kind >/dev/null 2>&1; then kind_cluster_name="${KUBE_CONTEXT#kind-}" - kind_images=("${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}") + kind_images=("${GATEWAY_IMAGE}") # The CI workflow loads its published sandbox archive before invoking this - # wrapper. Only load a sandbox image here when this script rebuilt it. - if [ "${reuse_sandbox_image}" != "1" ]; then - kind_images+=("${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}") + # wrapper. Load a replacement only when this script rebuilt or retagged it. + if [ "${reuse_sandbox_image}" != "1" ] \ + || [ "${SANDBOX_RUNTIME_IMAGE}" != "${sandbox_image}" ]; then + kind_images+=("${SANDBOX_RUNTIME_IMAGE}") fi # The CI workflow loads its published supervisor archive before invoking this # wrapper. Only load a supervisor image here when this script rebuilt it. - if [ "${reuse_supervisor_image}" != "1" ]; then - kind_images+=("${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}") + if [ "${reuse_supervisor_image}" != "1" ] \ + || [ "${SUPERVISOR_IMAGE}" != "${BUILD_SUPERVISOR_IMAGE}" ]; then + kind_images+=("${SUPERVISOR_IMAGE}") fi for image in "${kind_images[@]}"; do echo "Loading ${image} into kind cluster ${kind_cluster_name}..." @@ -1361,12 +1384,9 @@ else --namespace "${NAMESPACE}" --create-namespace \ "${helm_values_args[@]}" \ --set "fullnameOverride=openshell" \ - --set "image.repository=${REGISTRY_VALUE}/gateway" \ - --set "image.tag=${IMAGE_TAG_VALUE}" \ - --set "sandboxRuntime.image.repository=${REGISTRY_VALUE}/sandbox" \ - --set "sandboxRuntime.image.tag=${IMAGE_TAG_VALUE}" \ - --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ - --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ + "${GATEWAY_HELM_IMAGE_ARGS[@]}" \ + "${SUPERVISOR_HELM_IMAGE_ARGS[@]}" \ + "${SANDBOX_RUNTIME_HELM_IMAGE_ARGS[@]}" \ "${helm_extra_args[@]}" \ "${helm_post_renderer_args[@]}" \ --wait --timeout 5m diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 338217453c..6e8ec031ad 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -14,6 +14,11 @@ # HTTPS endpoint-only mode is intentionally unsupported here. Use a named # gateway config when mTLS materials are needed. # +# Supervisor image overrides: +# SUPERVISOR_IMAGE=... (common test-wrapper override) +# OPENSHELL_SUPERVISOR_IMAGE=... (existing compatibility override) +# SANDBOX_IMAGE=... (trusted sandbox runtime override) +# # Set OPENSHELL_E2E_PODMAN_STOP_TIMEOUT_SECS to override the managed gateway's # Podman sandbox stop timeout. The harness default is intentionally shorter # than the production driver default to keep CI teardown bounded. @@ -346,6 +351,16 @@ resolve_podman_supervisor_image() { return 0 fi + if [ -n "${SUPERVISOR_IMAGE:-}" ]; then + if [ -n "${CI:-}" ] && [ -z "${IMAGE_TAG:-}" ] \ + && ! e2e_image_reference_is_complete "${SUPERVISOR_IMAGE}"; then + echo "ERROR: IMAGE_TAG must be set in CI when SUPERVISOR_IMAGE is repository-only." >&2 + exit 2 + fi + printf '%s\n' "$(e2e_resolve_image_reference "${SUPERVISOR_IMAGE}" "${IMAGE_TAG:-dev}")" + return 0 + fi + if [ -n "${CI:-}" ]; then if [ -z "${IMAGE_TAG:-}" ]; then echo "ERROR: IMAGE_TAG must be set in CI when no Podman supervisor image override is provided." >&2 @@ -365,6 +380,10 @@ resolve_podman_sandbox_runtime_image() { printf '%s\n' "${OPENSHELL_SANDBOX_RUNTIME_IMAGE}" return 0 fi + if [ -n "${SANDBOX_IMAGE:-}" ]; then + printf '%s\n' "$(e2e_resolve_image_reference "${SANDBOX_IMAGE}" "${IMAGE_TAG:-dev}")" + return 0 + fi if [ -n "${CI:-}" ]; then if [ -z "${IMAGE_TAG:-}" ]; then @@ -387,6 +406,11 @@ ensure_podman_supervisor_image() { local dockerfile=${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor} local context="${WORKDIR}/supervisor-image" arch case "${image}" in + *@*) + echo "ERROR: supplied supervisor binaries cannot be built to a digest-pinned image reference: ${image}" >&2 + echo " Use a tagged image reference when building from OPENSHELL_E2E_SUPERVISOR_BIN." >&2 + exit 2 + ;; *:dev|*:latest) echo "ERROR: supplied supervisor binaries require a unique versioned image tag, not ${image}." >&2 exit 2 @@ -470,7 +494,7 @@ ensure_podman_supervisor_image() { fi echo "ERROR: supervisor image '${image}' is not available." >&2 - echo " Build it, push it, or set OPENSHELL_SUPERVISOR_IMAGE to a pullable image." >&2 + echo " Build it, push it, or set SUPERVISOR_IMAGE/OPENSHELL_SUPERVISOR_IMAGE to a pullable image." >&2 exit 2 } @@ -577,7 +601,11 @@ fi # isolated XDG store where this image was built. Address the local image by its # immutable manifest digest so policy=missing cannot resolve a mutable tag or # contact a registry for a different artifact. -SUPERVISOR_IMAGE_REPOSITORY="${SUPERVISOR_IMAGE%:*}" +SUPERVISOR_IMAGE_REPOSITORY="${SUPERVISOR_IMAGE%%@*}" +last_component="${SUPERVISOR_IMAGE_REPOSITORY##*/}" +if [[ "${last_component}" == *:* ]]; then + SUPERVISOR_IMAGE_REPOSITORY="${SUPERVISOR_IMAGE_REPOSITORY%:*}" +fi SUPERVISOR_RUNTIME_IMAGE="${SUPERVISOR_IMAGE_REPOSITORY}@${SUPERVISOR_IMAGE_DIGEST}" if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then echo "ERROR: supervisor runtime image is not digest-pinned: ${SUPERVISOR_RUNTIME_IMAGE}" >&2 diff --git a/examples/local-inference/README.md b/examples/local-inference/README.md index 9f5d8bc7cd..7cc36513ed 100644 --- a/examples/local-inference/README.md +++ b/examples/local-inference/README.md @@ -1,4 +1,4 @@ -# Provider-backed Inference Example +# Inference Example This example calls the NVIDIA API Catalog through its native OpenAI-compatible endpoint. OpenShell supplies endpoint-bound credentials and network policy from diff --git a/fern/docs.yml b/fern/docs.yml index fc280965db..1be5371527 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -61,9 +61,21 @@ versions: slug: dev availability: beta announcement: - message: 'OpenShell 0.1.0 is coming soon. Track progress in the 0.1.0 milestone, read the prerelease documentation, or install a prerelease.' + message: 'New in OpenShell 0.1.0: a stable release cadence, an improved security model, an expanded extension surface, and new APIs. Read the 0.1.0 upgrade guide.' redirects: + - source: "/openshell/latest/providers/google-vertex-ai" + destination: "/openshell/latest/providers/google-cloud#vertex-ai" + - source: "/openshell/dev/providers/google-vertex-ai" + destination: "/openshell/dev/providers/google-cloud#vertex-ai" + - source: "/openshell/latest/about/supported-agents" + destination: "/openshell/latest/about/run-an-agent" + - source: "/openshell/dev/about/supported-agents" + destination: "/openshell/dev/about/run-an-agent" + - source: "/openshell/latest/get-started/quickstart" + destination: "/openshell/latest/about/run-an-agent" + - source: "/openshell/dev/get-started/quickstart" + destination: "/openshell/dev/about/run-an-agent" - source: "/openshell/latest/sandboxes/providers-v2" destination: "/openshell/latest/providers/profiles" # Paths are relative to the site root; subpath prefix matches instances + custom-domain. diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index 321df6d6bc..0aec01e244 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -288,20 +288,24 @@ Shared implementations isolate each boundary's state and enforcement. Failure or ### Binary identity -The backend resolves executable identity for every accepted connection and delivers the result on `MediatedConnection` before network mediation evaluates policy. +The backend resolves executable identity for every accepted connection and delivers the result on `PendingTcpOpen` before network mediation evaluates policy. ```rust +struct ExecutableIdentity { + path: PathBuf, // absolute executable path + digest: Option, // bytes of the resolved executable object +} + struct BinaryIdentity { - binary_path: PathBuf, // absolute executable path - binary_digest: Option, // bytes of the resolved executable object - ancestors: Vec, // nearest first - cmdline_paths: Vec, // diagnostic context; never authorizes + executable: ExecutableIdentity, + ancestors: Vec, // nearest first + cmdline_paths: Vec, // diagnostic context; never authorizes } ``` -Identity describes the executable identity resolved for the accepted connection before policy evaluation. Paths are expressed in the workload's filesystem namespace. +Identity describes the executable identity resolved for the accepted connection before policy evaluation. Paths are expressed in the workload's filesystem namespace. Both the connection-owning executable and its ancestors can authorize access, so each carries integrity evidence as one inseparable path-and-digest value. Command-line paths are diagnostic context only and never authorize. -If binary identity cannot be resolved, the connection is denied. `ResolveError` reports that failure. A missing digest is represented as `None`. How a backend resolves identity is implementation-specific. +If binary identity cannot be resolved, the connection is denied. `ResolveError` reports that failure. A missing digest is represented as `None`; missing or conflicting evidence for any authorization-capable executable denies binary-scoped access. How a backend resolves identity is implementation-specific. Every identity field used for authorization is obtained by a trusted component from boundary or kernel state, rather than accepted as a workload claim. The result is bound to the active boundary and accepted connection; a transport tuple or workload-supplied identifier alone is not authoritative. Workload-supplied identity may be retained only as non-authorizing diagnostic context. If attribution is ambiguous or any required identity field cannot be established, the connection is denied. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 16c298ddd1..d2b2076a9c 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -99,7 +99,8 @@ and PriorityClasses need matching administrator-owned labels; namespace membership and read-only access do not grant approval. GPU devices and operator-selected image-pull Secrets do not need admission labels. In managed mode, inspect the configured source image-pull Secret in the gateway namespace -and the gateway-owned copy in the workspace namespace. Legacy workloads without +and the generation copies (`openshell.ai/component=image-pull`) in the workspace +namespace. Legacy workloads without admission provenance need recreation. Do not automatically label control-plane resources or disable enforcement as a repair. @@ -134,6 +135,19 @@ bundle is available; Kubernetes projects its bundle through a Secret. Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. First-party standalone drivers require the socket parent directory to be owned by the driver's effective UID, force its mode to `0700`, create the socket with mode `0600`, and accept only peers with that same UID. Check the parent and socket separately with `stat`; a gateway running under a different UID cannot connect even when filesystem permissions or group membership would otherwise allow it. Operator-supplied drivers must provide equivalent access control appropriate to their implementation. Check gateway logs for connection errors, `GetCapabilities` failures, missing peer metadata, protocol-major mismatch, unmet required capabilities, or an unexpected advertised driver name. `openshell gateway info` reports successful startup negotiations. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. +For the Kubernetes Secrets credential driver, every provider credential lives in +the configured `namespace`, in every workspace mode. A `PermissionDenied` error +naming another namespace means the provider's credential handle points outside +the configured namespace; recreate the provider. An `unknown field` startup +error for `[openshell.credential_drivers.kubernetes-secrets]` means the table +sets a key the driver does not accept. Confirm the gateway can reach the +credential namespace: + +```bash +kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -A3 '^\[openshell\.credential_drivers\.kubernetes-secrets\]' +kubectl auth can-i get secrets -n --as system:serviceaccount:openshell:openshell +``` + For a configured Vault credential driver, inspect its endpoint and trust bundle before debugging provider resolution. Non-loopback addresses must use HTTPS, and the driver never follows redirects. A private CA bundle augments platform @@ -752,13 +766,14 @@ Do not suspend or delete the workload Pod manually. The driver advances to workload. If a Sandbox remains in the `suspending` bootstrap phase, verify that the -gateway ServiceAccount can create, list, and delete Secrets in the sandbox -namespace. Recovery lists generation Secrets by sandbox and component labels -even when none remain, then deletes stale entries with UID preconditions before -clearing the suspension annotations: +gateway ServiceAccount can create and delete Secrets in the sandbox +namespace. In operator mode, those permissions come only from the +`openshell-workspace` chart installed in the namespace. Recovery deletes the +recorded generation's bootstrap Secrets by name before clearing the suspension +annotations: ```bash -for verb in create list delete; do +for verb in create delete; do kubectl auth can-i "$verb" secrets \ --namespace \ --as system:serviceaccount:openshell:openshell diff --git a/tasks/scripts/test-e2e-image-overrides.sh b/tasks/scripts/test-e2e-image-overrides.sh new file mode 100755 index 0000000000..83c92d0012 --- /dev/null +++ b/tasks/scripts/test-e2e-image-overrides.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=e2e/support/gateway-common.sh +source "${ROOT}/e2e/support/gateway-common.sh" + +assert_resolves() { + local description=$1 + local expected=$2 + shift 2 + local actual + actual="$(e2e_resolve_image_reference "$@")" + if [ "${actual}" != "${expected}" ]; then + echo "FAIL: ${description}: expected '${expected}', got '${actual}'" >&2 + exit 1 + fi +} + +assert_resolves "repository inherits tag" \ + "registry.example/gateway:test" \ + "registry.example/gateway" test +assert_resolves "repository trims trailing slash" \ + "registry.example/gateway:test" \ + "registry.example/gateway/" test +assert_resolves "tagged reference is unchanged" \ + "registry.example/gateway:branch" \ + "registry.example/gateway:branch" test +assert_resolves "digest reference is unchanged" \ + "registry.example/gateway@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "registry.example/gateway@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" test +assert_resolves "registry port inherits tag" \ + "localhost:5000/openshell/gateway:test" \ + "localhost:5000/openshell/gateway" test + +if e2e_image_reference_is_complete "registry.example/gateway:branch" \ + && e2e_image_reference_is_complete "registry.example/gateway@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + && ! e2e_image_reference_is_complete "registry.example/gateway" \ + && e2e_image_reference_has_digest "registry.example/gateway@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + && ! e2e_image_reference_has_digest "registry.example/gateway:branch"; then + : +else + echo "FAIL: image reference detection" >&2 + exit 1 +fi + +assert_reference_part() { + local description=$1 + local expected=$2 + local actual=$3 + if [ "${actual}" != "${expected}" ]; then + echo "FAIL: ${description}: expected '${expected}', got '${actual}'" >&2 + exit 1 + fi +} + +assert_reference_part "repository strips tag" \ + "registry.example/gateway" \ + "$(e2e_image_reference_repository "registry.example/gateway:branch")" +assert_reference_part "repository preserves registry port" \ + "localhost:5000/openshell/gateway" \ + "$(e2e_image_reference_repository "localhost:5000/openshell/gateway:test")" +assert_reference_part "registry extracts hostname" \ + "registry.example" \ + "$(e2e_image_reference_registry "registry.example/openshell/gateway:branch")" +assert_reference_part "registry extracts hostname and port" \ + "localhost:5000" \ + "$(e2e_image_reference_registry "localhost:5000/openshell/gateway:test")" +assert_reference_part "repository path excludes registry" \ + "openshell/gateway" \ + "$(e2e_image_reference_repository_path "registry.example/openshell/gateway:branch")" +assert_reference_part "tag extracts tag" \ + "branch" \ + "$(e2e_image_reference_tag "registry.example/gateway:branch")" +assert_reference_part "digest extracts digest" \ + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "$(e2e_image_reference_digest "registry.example/gateway@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")" +assert_reference_part "digest has no tag" \ + "" \ + "$(e2e_image_reference_tag "registry.example/gateway@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")" + +assert_helm_image_translation() { + local description=$1 + local image=$2 + local expected_registry=$3 + local expected_repository=$4 + local expected_tag=$5 + local expected_digest=$6 + + assert_reference_part "${description} registry" "${expected_registry}" \ + "$(e2e_image_reference_registry "${image}")" + assert_reference_part "${description} repository" "${expected_repository}" \ + "$(e2e_image_reference_repository_path "${image}")" + assert_reference_part "${description} tag" "${expected_tag}" \ + "$(e2e_image_reference_tag "${image}")" + assert_reference_part "${description} digest" "${expected_digest}" \ + "$(e2e_image_reference_digest "${image}")" +} + +# The Kubernetes wrapper passes these three fields directly to Helm. Exercise +# a mixed tagged/digest-pinned set so each independently configurable image is +# translated without embedding a complete reference in image.repository. +assert_helm_image_translation "gateway" \ + "registry.example/gateway:branch" "registry.example" "gateway" "branch" "" +assert_helm_image_translation "supervisor" \ + "localhost:5000/openshell/supervisor:test" "localhost:5000" "openshell/supervisor" "test" "" +assert_helm_image_translation "sandbox" \ + "registry.example/sandbox@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" \ + "registry.example" "sandbox" "" "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +echo "E2E image override tests passed." diff --git a/tasks/test.toml b/tasks/test.toml index 701bab7327..de65f527d6 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -13,6 +13,7 @@ depends = [ "test:install-sh", "test:build-env", "test:gateway-pull-policy", + "test:e2e-image-overrides", "test:gateway-config", "test:e2e-parity", "test:packaging-assets", @@ -50,6 +51,12 @@ run = "tasks/scripts/test-gateway-pull-policy.sh" run_windows = "echo Skipping test:gateway-pull-policy: Unix gateway scripts do not apply on Windows." hide = true +["test:e2e-image-overrides"] +description = "Test E2E gateway and supervisor image override resolution" +run = "tasks/scripts/test-e2e-image-overrides.sh" +run_windows = "echo Skipping test:e2e-image-overrides: Unix E2E wrappers do not apply on Windows." +hide = true + ["test:packaging-assets"] description = "Run static packaging asset tests" run = "tasks/scripts/test-packaging-assets.sh" @@ -225,7 +232,7 @@ run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debu ["e2e:kubernetes:workspace-managed"] description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } +env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_CREDENTIAL_DRIVER = "kubernetes-secrets", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"