From acbac9cb795094986cbab016bfd0352d980b17f6 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 29 Sep 2026 02:18:30 +0000 Subject: [PATCH 1/3] feat(sandbox): add main restart policy (#2798) * feat(sandbox): add main restart policy Signed-off-by: Drew Newberry * fix(sandbox): harden policy-driven restarts Signed-off-by: Drew Newberry * fix(sandbox): address restart review feedback Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com> * fix(sandbox): port restart policy to current runtime Signed-off-by: Drew Newberry * perf(sandbox): restart promptly after terminal delivery Signed-off-by: Drew Newberry --------- Signed-off-by: Drew Newberry Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com> --- .agents/skills/tui-development/SKILL.md | 8 +- architecture/compute-runtimes.md | 19 + architecture/sandbox.md | 13 +- crates/openshell-cli/src/main.rs | 55 + crates/openshell-cli/src/run.rs | 121 +- .../openshell-driver-podman/src/container.rs | 15 + crates/openshell-driver-vm/src/driver.rs | 72 +- crates/openshell-sdk/src/client.rs | 7 + crates/openshell-sdk/src/lib.rs | 10 +- crates/openshell-sdk/src/types.rs | 45 +- crates/openshell-server/src/compute/mod.rs | 1536 +++++++++++++-- crates/openshell-server/src/grpc/sandbox.rs | 34 +- .../openshell-server/src/grpc/validation.rs | 25 +- crates/openshell-server/src/lib.rs | 8 +- crates/openshell-server/src/storage_proto.rs | 22 +- crates/openshell-tui/src/app.rs | 12 + crates/openshell-tui/src/lib.rs | 37 +- crates/openshell-tui/src/ui/sandbox_detail.rs | 30 +- docs/how-it-works/sandboxes/overview.mdx | 28 +- docs/how-it-works/sandboxes/runtimes.mdx | 5 + e2e/rust/tests/sandbox_lifecycle.rs | 66 + proto/openshell.proto | 26 +- python/openshell/sandbox.py | 10 + python/openshell/sandbox_test.py | 6 + .../v1/internal/converter/coverage_test.go | 22 +- .../v1/internal/converter/sandbox.go | 37 +- .../v1/internal/converter/sandbox_test.go | 26 +- sdk/go/openshell/v1/types.go | 10 + sdk/go/openshell/v1/types/sandbox.go | 10 +- sdk/go/openshell/v1/types/types.go | 10 + sdk/go/proto/openshellv1/openshell.pb.go | 1664 +++++++++-------- sdk/typescript/src/client.test.ts | 39 +- sdk/typescript/src/client.ts | 26 + sdk/typescript/src/index.ts | 1 + skills/debug-openshell-cluster/SKILL.md | 8 + skills/openshell-cli/SKILL.md | 11 +- 36 files changed, 3072 insertions(+), 1002 deletions(-) diff --git a/.agents/skills/tui-development/SKILL.md b/.agents/skills/tui-development/SKILL.md index 055884e1ed..832bb75144 100644 --- a/.agents/skills/tui-development/SKILL.md +++ b/.agents/skills/tui-development/SKILL.md @@ -102,7 +102,7 @@ match app.screen { } ``` -Within the `Sandbox` screen, the top 20% renders sandbox metadata (`sandbox_detail`), and the bottom 80% dispatches based on focus and tab state: +Within the `Sandbox` screen, `sandbox_detail::required_height` sizes the metadata and restart status pane to its contents. The remaining area dispatches based on focus and tab state: ```rust match app.focus { @@ -262,7 +262,7 @@ The `Theme` struct has 16 `Style` fields, accessed at runtime via `app.theme`: | `border` | EVERGLADE fg | Light sage fg | Unfocused panel borders | | `border_focused` | NVIDIA_GREEN fg | NVIDIA_GREEN_DARK fg | Focused panel borders | | `status_ok` | NVIDIA_GREEN fg | NVIDIA_GREEN_DARK fg | Healthy, INFO, Ready | -| `status_warn` | Yellow fg | Dark yellow fg | Degraded, WARN, Provisioning | +| `status_warn` | Yellow fg | Dark yellow fg | Degraded, WARN, Provisioning, Starting | | `status_err` | Red fg | Dark red fg | Unhealthy, ERROR | | `key_hint` | NVIDIA_GREEN fg | NVIDIA_GREEN_DARK fg | Keyboard shortcut labels | | `log_cursor` | EVERGLADE bg | Light green bg | Selected log line highlight | @@ -297,7 +297,7 @@ fn draw_detail_popup(frame: &mut Frame<'_>, data: &MyData, area: Rect, theme: &T - **Selected row**: Green `▌` left-border marker on the selected row. Active gateway also gets a green `●` dot. - **Focused panel**: Border changes from `border` to `border_focused` style. -- **Status indicators**: Green for healthy/ready/info, yellow for degraded/provisioning/warn, red for unhealthy/error. +- **Status indicators**: Green for healthy/ready/info, yellow for degraded/provisioning/starting/warn, red for unhealthy/error. - **Separators**: Muted `│` characters between title bar segments and nav bar sections. - **Log source labels**: `"sandbox"` source renders in `accent` (green), `"gateway"` in `muted`. @@ -423,7 +423,7 @@ All actions are accessible via keyboard shortcuts displayed in the nav bar. The | `crates/openshell-tui/src/ui/providers.rs` | Provider list table with profile-aware columns: Name, Category, Type, Credentials, Workspace | | `crates/openshell-tui/src/ui/global_settings.rs` | Global settings table: Key, Type, Value. Includes edit overlay, confirm-set, and confirm-delete popups | | `crates/openshell-tui/src/ui/sandboxes.rs` | Reusable sandbox table widget with columns: Name, Status, Created, Age, Image, Workspace, Notes | -| `crates/openshell-tui/src/ui/sandbox_detail.rs` | Sandbox metadata view — name, status, image, created, age, providers, policy version | +| `crates/openshell-tui/src/ui/sandbox_detail.rs` | Sandbox metadata view — name, status, image, created, age, restart policy/status, providers, policy version | | `crates/openshell-tui/src/ui/sandbox_policy.rs` | Policy viewer — rendered policy lines with scroll support, tab title | | `crates/openshell-tui/src/ui/sandbox_settings.rs` | Sandbox settings table: Key, Type, Value, Scope. Includes edit overlay and confirm popups | | `crates/openshell-tui/src/ui/sandbox_logs.rs` | Structured log viewer — timestamp, source, level, target, message, key=value fields, scroll position, source filter, visual selection mode, clipboard copy | diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index ec6fff115a..715389d204 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -75,6 +75,14 @@ Canonical main-process support is part of the `ComputeDriver` contract. Every in-tree and extension driver must forward the exact specification; it is not an optional capability that drivers can omit or negotiate. +The gateway also owns canonical main-process restart policy. Drivers disable +native container, pod, or VM restart behavior and implement the existing stop +and start operations. When policy selects a restart, the gateway persists the +`Starting` phase and backoff deadline, stops compute, starts it again after the +deadline, and waits for a new supervisor session before returning to `Ready`. +Restart count, exit code, and deadline distinguish this path from an explicit +start. Driver snapshots cannot override that gateway-owned transition. + Drivers own runtime-specific platform event interpretation. When an event should drive client provisioning UI, the driver attaches the shared `openshell.progress.*` metadata defined in `openshell-core` instead of requiring @@ -90,6 +98,9 @@ produce the public `SandboxPhase`: backend_phase = derive_phase(driver_status) public_phase = + if persisted_phase == Stopped: → Stopped + if persisted_phase == Stopping and backend not stopped: → Stopping + if persisted_phase == Starting and backend not Ready: → Starting if backend_phase in {Error, Deleting}: → pass through (terminal precedence) if driver_reports_runtime_readiness && backend_phase == Ready: → Ready if backend_phase == Ready && session connected: → Ready @@ -193,6 +204,14 @@ restart a completed or failed canonical process. Before an explicit restart, the gateway disconnects the prior supervisor session and deletes its SSH sessions so credentials cannot cross runtime generations. +When the sandbox restart policy selects replacement, the gateway instead +persists `Starting` with the previous exit result, restart count, and backoff +deadline. The leader claims each due attempt before stopping compute, rotates +launch credentials, starts the driver with the persisted generation, and waits +for a different supervisor instance. Stop and delete intent fence late driver +starts. A fresh supervisor clears the deadline and returns the sandbox to +`Ready`; a successful main process exit under `OnFailure` remains `Completed`. + `StopSandbox` and `StartSandbox` are idempotent driver operations. Stop retains the driver resource and its persistent workspace boundary while making exec, SSH, forwarding, and exposed services unavailable. Start reactivates the diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 7dce15d6ab..a704b25341 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -919,9 +919,16 @@ the runtime policy it would enforce. for that finalized supervisor session to disconnect before deleting an ephemeral sandbox. Exit code 0 records `Completed/MainProcessCompleted`; nonzero and signal-normalized exits record - `Error/MainProcessFailed`. Infrastructure failures also use `Error`, with a - distinct condition reason and no fabricated canonical-process result. Runtime - restart policies must not replace the canonical process. + `Error/MainProcessFailed` when the gateway policy is `Never`. `OnFailure` + replaces the runtime after a nonzero result; `Always` replaces it after any + result. The gateway starts the first replacement after terminal delivery, then + persists a 1-second doubling backoff for repeated quick exits, capped at 3 + minutes. A 10-second run resets the count. A bounded fallback recovers a + restart when terminal finalization is lost. Each replacement starts a new + runtime generation with fresh launch credentials. Infrastructure failures also + use `Error`, with a distinct condition reason and no fabricated canonical-process + result. + Native runtime restart policies remain disabled. ## Shared Boundary Primitives diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 47f537faf7..f656fd8833 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1535,6 +1535,14 @@ enum SandboxCommands { #[arg(long, conflicts_with = "editor")] detach: bool, + /// Restart behavior after the canonical main process exits. + #[arg( + long, + value_parser = ["never", "on-failure", "always"], + default_value = "never" + )] + restart_policy: String, + /// Auto-create missing providers from local credentials. /// /// Without this flag, an interactive prompt asks per-provider; @@ -3314,6 +3322,7 @@ async fn run_async() -> Result<()> { tty, no_tty, detach, + restart_policy, auto_providers, no_auto_providers, labels, @@ -3416,6 +3425,7 @@ async fn run_async() -> Result<()> { output: output.as_str(), detach, suppress_credential_warnings: no_credential_warnings, + restart_policy: &restart_policy, }, &cli.workspace, &tls, @@ -6008,6 +6018,51 @@ mod tests { } } + #[test] + fn sandbox_create_restart_policy_defaults_to_never() { + let cli = Cli::try_parse_from(["openshell", "sandbox", "create"]).unwrap(); + match cli.command { + Some(Commands::Sandbox { + command: Some(SandboxCommands::Create { restart_policy, .. }), + .. + }) => assert_eq!(restart_policy, "never"), + other => panic!("expected SandboxCommands::Create, got: {other:?}"), + } + } + + #[test] + fn sandbox_create_restart_policy_accepts_on_failure() { + let cli = Cli::try_parse_from([ + "openshell", + "sandbox", + "create", + "--restart-policy", + "on-failure", + ]) + .unwrap(); + match cli.command { + Some(Commands::Sandbox { + command: Some(SandboxCommands::Create { restart_policy, .. }), + .. + }) => assert_eq!(restart_policy, "on-failure"), + other => panic!("expected SandboxCommands::Create, got: {other:?}"), + } + } + + #[test] + fn sandbox_create_restart_policy_rejects_unknown_value() { + assert!( + Cli::try_parse_from([ + "openshell", + "sandbox", + "create", + "--restart-policy", + "unless-stopped", + ]) + .is_err() + ); + } + #[test] fn sandbox_create_detach_parses_with_main_command() { let cli = Cli::try_parse_from([ diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 15d91b5891..a94ea9fb9d 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -56,11 +56,11 @@ use openshell_core::proto::{ ListSandboxPoliciesRequest, ListSandboxTemplatesRequest, ListSandboxesRequest, ListServicesRequest, PolicySource, PolicyStatus, RejectDraftChunkRequest, ResourceRequirements, RevokeSshSessionRequest, Sandbox, SandboxCondition, SandboxPhase, SandboxPolicy, - SandboxResources, SandboxServiceExposure, SandboxServiceLevel, SandboxSpec, SandboxStartup, - SandboxTemplate, SandboxWorkloadConfig, SandboxWorkloadTemplate, SandboxWorkloadTemplateSpec, - ServiceEndpointResponse, SettingScope, StartSandboxRequest, StopSandboxRequest, - TcpForwardFrame, TcpForwardInit, TcpRelayTarget, UpdateConfigRequest, WatchSandboxRequest, - exec_sandbox_event, tcp_forward_init, + SandboxResources, SandboxRestartPolicy, SandboxServiceExposure, SandboxServiceLevel, + SandboxSpec, SandboxStartup, SandboxTemplate, SandboxWorkloadConfig, SandboxWorkloadTemplate, + SandboxWorkloadTemplateSpec, ServiceEndpointResponse, SettingScope, StartSandboxRequest, + StopSandboxRequest, TcpForwardFrame, TcpForwardInit, TcpRelayTarget, UpdateConfigRequest, + WatchSandboxRequest, exec_sandbox_event, tcp_forward_init, }; use openshell_core::settings; use openshell_core::{ObjectId, ObjectName, ObjectWorkspace}; @@ -452,6 +452,7 @@ pub struct SandboxCreateConfig<'a> { pub output: &'a str, pub detach: bool, pub suppress_credential_warnings: bool, + pub restart_policy: &'a str, } impl Default for SandboxCreateConfig<'_> { @@ -480,6 +481,7 @@ impl Default for SandboxCreateConfig<'_> { output: "table", detach: false, suppress_credential_warnings: false, + restart_policy: "never", } } } @@ -516,6 +518,7 @@ pub async fn sandbox_create( output, detach, suppress_credential_warnings, + restart_policy, } = config; if editor.is_some() && !command.is_empty() { @@ -670,6 +673,12 @@ pub async fn sandbox_create( template: inline_template, command: main_command, tty: main_terminal, + restart_policy: match restart_policy { + "never" => SandboxRestartPolicy::Never as i32, + "on-failure" => SandboxRestartPolicy::OnFailure as i32, + "always" => SandboxRestartPolicy::Always as i32, + value => return Err(miette::miette!("invalid restart policy '{value}'")), + }, ..SandboxSpec::default() }), name: name.unwrap_or_default().to_string(), @@ -1697,6 +1706,41 @@ where "Resource version:".dimmed(), sandbox.metadata.as_ref().map_or(0, |m| m.resource_version) ); + println!( + " {} {}", + "Restart policy:".dimmed(), + sandbox + .spec + .as_ref() + .map_or("never", |spec| { restart_policy_name(spec.restart_policy) }) + ); + if let Some(status) = sandbox.status.as_ref() { + println!( + " {} {}", + "Main process instance:".dimmed(), + if status.main_process_instance_id.is_empty() { + "-" + } else { + &status.main_process_instance_id + } + ); + println!( + " {} {}", + "Last exit code:".dimmed(), + status + .exit_code + .map_or_else(|| "-".to_string(), |code| code.to_string()) + ); + println!(" {} {}", "Restart count:".dimmed(), status.restart_count); + println!( + " {} {}", + "Next restart:".dimmed(), + status.next_restart_time.as_ref().map_or_else( + || "-".to_string(), + |time| format_epoch_ms(proto_timestamp_ms(Some(time))), + ) + ); + } // Display labels if present if let Some(metadata) = &sandbox.metadata @@ -2846,6 +2890,14 @@ fn endpoint_status_display_lines(endpoint: &EndpointStatus) -> Vec { ] } +fn restart_policy_name(policy: i32) -> &'static str { + match SandboxRestartPolicy::try_from(policy) { + Ok(SandboxRestartPolicy::OnFailure) => "on-failure", + Ok(SandboxRestartPolicy::Always) => "always", + Ok(SandboxRestartPolicy::Unspecified | SandboxRestartPolicy::Never) | Err(_) => "never", + } +} + fn sandbox_detail_to_json( sandbox: &Sandbox, config: &GetSandboxConfigResponse, @@ -2855,6 +2907,33 @@ fn sandbox_detail_to_json( .as_object_mut() .expect("sandbox_to_json returns object"); + let restart_policy = sandbox + .spec + .as_ref() + .map_or("never", |spec| restart_policy_name(spec.restart_policy)); + obj.insert("restart_policy".into(), serde_json::json!(restart_policy)); + if let Some(status) = sandbox.status.as_ref() { + obj.insert( + "main_process_instance_id".into(), + serde_json::json!(status.main_process_instance_id), + ); + obj.insert("exit_code".into(), serde_json::json!(status.exit_code)); + obj.insert( + "restart_count".into(), + serde_json::json!(status.restart_count), + ); + obj.insert( + "next_restart_at_ms".into(), + serde_json::json!(proto_timestamp_ms(status.next_restart_time.as_ref())), + ); + obj.insert( + "main_process_started_at_ms".into(), + serde_json::json!(proto_timestamp_ms( + status.main_process_started_time.as_ref() + )), + ); + } + let policy_source = if config.policy_source == PolicySource::Global as i32 { "global" } else { @@ -6440,10 +6519,10 @@ mod tests { use openshell_core::proto::{ EndpointResult, EndpointStatus, GetSandboxConfigResponse, GpuResourceRequirements, PolicySource, PolicyStatus, ResourceRequirements, Sandbox, SandboxCondition, SandboxPhase, - SandboxPolicy, SandboxPolicyRevision, SandboxResources, SandboxStatus, - SandboxWorkloadConfig, SandboxWorkloadTemplate, SandboxWorkloadTemplateProvenance, - SandboxWorkloadTemplateSpec, ServiceEndpoint, ServiceEndpointResponse, WorkspaceMember, - WorkspaceRole, datamodel::v1::ObjectMeta, + SandboxPolicy, SandboxPolicyRevision, SandboxResources, SandboxRestartPolicy, SandboxSpec, + SandboxStatus, SandboxWorkloadConfig, SandboxWorkloadTemplate, + SandboxWorkloadTemplateProvenance, SandboxWorkloadTemplateSpec, ServiceEndpoint, + ServiceEndpointResponse, WorkspaceMember, WorkspaceRole, datamodel::v1::ObjectMeta, }; #[test] @@ -7717,6 +7796,22 @@ mod tests { }; sandbox.set_phase(SandboxPhase::Ready as i32); sandbox.set_current_policy_version(2); + sandbox.spec = Some(SandboxSpec { + restart_policy: SandboxRestartPolicy::OnFailure as i32, + ..Default::default() + }); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + main_process_instance_id: "main-2".to_string(), + exit_code: Some(9), + restart_count: 2, + next_restart_time: openshell_core::time::timestamp_from_millis(1_700_000_000_000).ok(), + main_process_started_time: openshell_core::time::timestamp_from_millis( + 1_699_999_000_000, + ) + .ok(), + ..Default::default() + }); let config = GetSandboxConfigResponse { policy_source: PolicySource::Global as i32, @@ -7728,7 +7823,13 @@ mod tests { assert_eq!(json["id"], "sb-123"); assert_eq!(json["name"], "test-sb"); - assert_eq!(json["phase"], "Ready"); + assert_eq!(json["phase"], "Starting"); + assert_eq!(json["restart_policy"], "on-failure"); + assert_eq!(json["main_process_instance_id"], "main-2"); + assert_eq!(json["exit_code"], 9); + assert_eq!(json["restart_count"], 2); + assert_eq!(json["next_restart_at_ms"], 1_700_000_000_000_i64); + assert_eq!(json["main_process_started_at_ms"], 1_699_999_000_000_i64); assert_eq!(json["policy_source"], "global"); assert_eq!(json["revision"], 3); assert!(json["policy"].is_null()); diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index bb4ee88c9f..3092eb73b3 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -263,6 +263,11 @@ pub struct ContainerSpec { stop_timeout: u32, /// Extra /etc/hosts entries for the networked supervisor container. /// The isolated workload resolves host aliases through policy DNS. + /// Native restart stays disabled; the gateway owns sandbox restart policy. + restart_policy: String, + /// Extra /etc/hosts entries. Used to inject `host.containers.internal` + /// via Podman's `host-gateway` magic so sandbox containers can reach + /// the gateway server running on the host in rootless mode. hostadd: Vec, /// Search domains written to `/etc/resolv.conf` by Podman. dns_search: Vec, @@ -1283,6 +1288,10 @@ fn build_base_spec( // Inject stable host aliases into the networked supervisor container. // The workload clears these entries and resolves the driver-neutral // alias through the policy-DNS relay instead. + restart_policy: "no".to_string(), + // Inject stable host aliases into /etc/hosts so sandbox containers can + // reach services on the host. `host.openshell.internal` is the driver- + // neutral alias used by policies and e2e tests. hostadd: hostadd_entries(config), // Preserve Podman's resolver defaults for both policy-DNS and ordinary // sandboxes. Namespace-local capture supports UDP and TCP, so it must @@ -3345,6 +3354,12 @@ mod tests { config.host_gateway_ip = "192.168.127.254".to_string(); let spec = build_container_spec(&sandbox, &config); + assert_eq!( + spec["restart_policy"].as_str(), + Some("no"), + "the gateway owns sandbox restart policy" + ); + let hostadd: Vec<&str> = spec["hostadd"] .as_array() .expect("hostadd should be an array") diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 8cdabb22c5..58ad21892e 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -2315,21 +2315,15 @@ impl VmDriver { ); } - if clear_stop_marker { - match tokio::fs::remove_file(state_dir.join(SANDBOX_STOPPED_FILE)).await { - Ok(()) => {} - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => { - self.registry.lock().await.remove(&sandbox.id); - warn!( - sandbox_id = %sandbox.id, - state_dir = %state_dir.display(), - error = %err, - "vm driver: cannot clear stop marker for persisted sandbox restore" - ); - return false; - } - } + if clear_stop_marker && let Err(err) = clear_explicit_start_markers(&state_dir).await { + self.registry.lock().await.remove(&sandbox.id); + warn!( + sandbox_id = %sandbox.id, + state_dir = %state_dir.display(), + error = %err, + "vm driver: cannot clear lifecycle markers for persisted sandbox restore" + ); + return false; } self.publish_platform_event( @@ -6214,6 +6208,19 @@ async fn remove_runtime_generation_material(state_dir: &Path) -> Result<(), Stri Ok(()) } +async fn clear_explicit_start_markers(state_dir: &Path) -> Result<(), std::io::Error> { + // Remove the terminal marker first. If clearing the durable stop marker + // fails, the sandbox remains stopped and a later start can safely retry. + for marker in [MAIN_PROCESS_EXITED_FILE, SANDBOX_STOPPED_FILE] { + match tokio::fs::remove_file(state_dir.join(marker)).await { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + } + Ok(()) +} + #[cfg(unix)] async fn restrict_owner_read_write(path: &Path) -> Result<(), std::io::Error> { tokio::fs::set_permissions(path, fs::Permissions::from_mode(0o600)).await @@ -8735,6 +8742,9 @@ mod tests { tokio::fs::write(state_dir.join(SANDBOX_STOPPED_FILE), b"stopped\n") .await .unwrap(); + tokio::fs::write(state_dir.join(MAIN_PROCESS_EXITED_FILE), b"terminal\n") + .await + .unwrap(); let snapshot = sandbox_snapshot(&sandbox, stopped_condition(), false); driver.registry.lock().await.insert( sandbox.id.clone(), @@ -8766,6 +8776,12 @@ mod tests { .is_ok(), "failed start must retain its durable stop marker" ); + assert!( + tokio::fs::metadata(state_dir.join(MAIN_PROCESS_EXITED_FILE)) + .await + .is_ok(), + "failed start must retain its terminal marker" + ); let restored = driver .get_sandbox(&sandbox.id, &sandbox.name) .await @@ -8872,6 +8888,32 @@ mod tests { ) } + #[tokio::test] + async fn explicit_start_clears_stop_and_terminal_markers() { + let temp = tempfile::tempdir().unwrap(); + let state_dir = temp.path().join("sandbox"); + tokio::fs::create_dir_all(&state_dir).await.unwrap(); + tokio::fs::write(state_dir.join(SANDBOX_STOPPED_FILE), b"stopped\n") + .await + .unwrap(); + tokio::fs::write(state_dir.join(MAIN_PROCESS_EXITED_FILE), b"terminal\n") + .await + .unwrap(); + + clear_explicit_start_markers(&state_dir).await.unwrap(); + + assert!( + !tokio::fs::try_exists(state_dir.join(SANDBOX_STOPPED_FILE)) + .await + .unwrap() + ); + assert!( + !tokio::fs::try_exists(state_dir.join(MAIN_PROCESS_EXITED_FILE)) + .await + .unwrap() + ); + } + #[test] fn prepare_sandbox_overlay_preserves_existing_overlay_on_start() { let base = unique_temp_dir(); diff --git a/crates/openshell-sdk/src/client.rs b/crates/openshell-sdk/src/client.rs index 0bf2412282..081ce94f89 100644 --- a/crates/openshell-sdk/src/client.rs +++ b/crates/openshell-sdk/src/client.rs @@ -1327,6 +1327,7 @@ fn create_sandbox_request(spec: SandboxSpec) -> proto::CreateSandboxRequest { command, tty, service_exposures, + restart_policy, } = spec; let template = image.map(|image| proto::SandboxTemplate { image, @@ -1344,6 +1345,7 @@ fn create_sandbox_request(spec: SandboxSpec) -> proto::CreateSandboxRequest { resource_requirements, command, tty, + restart_policy: proto::SandboxRestartPolicy::from(restart_policy) as i32, ..proto::SandboxSpec::default() }), name: name.unwrap_or_default(), @@ -1634,11 +1636,16 @@ mod tests { let request = create_sandbox_request(SandboxSpec { command: vec!["/opt/agent binary".into(), "--serve exactly".into()], tty: false, + restart_policy: crate::types::SandboxRestartPolicy::OnFailure, ..SandboxSpec::default() }); let spec = request.spec.expect("sandbox spec should be present"); assert_eq!(spec.command, ["/opt/agent binary", "--serve exactly"]); assert!(!spec.tty); + assert_eq!( + spec.restart_policy(), + proto::SandboxRestartPolicy::OnFailure + ); } } diff --git a/crates/openshell-sdk/src/lib.rs b/crates/openshell-sdk/src/lib.rs index 269104ae2d..2d79b28cc6 100644 --- a/crates/openshell-sdk/src/lib.rs +++ b/crates/openshell-sdk/src/lib.rs @@ -51,9 +51,9 @@ pub use pagination::{Page, Pager}; pub use refresh::{Refresh, RefreshError, RefreshedToken, TokenSource}; pub use types::{ DeleteOptions, DeletionOutcome, DeletionResult, ExecOptions, ExecResult, Health, ListOptions, - LogLine, PlatformEvent, SandboxPhase, SandboxRef, SandboxResources, SandboxServiceLevel, - SandboxSpec, SandboxStartup, SandboxTemplateCreateSpec, SandboxTemplateListOptions, - SandboxWorkloadConfig, SandboxWorkloadTemplate, SandboxWorkloadTemplateProvenance, - SandboxWorkloadTemplateSpec, ServiceExposure, ServiceStatus, WatchEvent, WatchOptions, - WorkspaceRef, + LogLine, PlatformEvent, SandboxPhase, SandboxRef, SandboxResources, SandboxRestartPolicy, + SandboxServiceLevel, SandboxSpec, SandboxStartup, SandboxTemplateCreateSpec, + SandboxTemplateListOptions, SandboxWorkloadConfig, SandboxWorkloadTemplate, + SandboxWorkloadTemplateProvenance, SandboxWorkloadTemplateSpec, ServiceExposure, ServiceStatus, + WatchEvent, WatchOptions, WorkspaceRef, }; diff --git a/crates/openshell-sdk/src/types.rs b/crates/openshell-sdk/src/types.rs index 57bed05e08..b7297a3fee 100644 --- a/crates/openshell-sdk/src/types.rs +++ b/crates/openshell-sdk/src/types.rs @@ -240,6 +240,25 @@ impl From for SandboxPhase { } } +/// Gateway policy for replacing the canonical main process after it exits. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum SandboxRestartPolicy { + #[default] + Never, + OnFailure, + Always, +} + +impl From for proto::SandboxRestartPolicy { + fn from(value: SandboxRestartPolicy) -> Self { + match value { + SandboxRestartPolicy::Never => Self::Never, + SandboxRestartPolicy::OnFailure => Self::OnFailure, + SandboxRestartPolicy::Always => Self::Always, + } + } +} + /// Caller intent for a new sandbox. /// /// Only the most commonly used fields are exposed. Callers that need the @@ -266,6 +285,8 @@ pub struct SandboxSpec { pub tty: bool, /// Loopback HTTP services to expose when the sandbox is created. pub service_exposures: Vec, + /// Restart behavior after the canonical main process exits. + pub restart_policy: SandboxRestartPolicy, } /// A loopback HTTP service to expose during sandbox creation. @@ -346,6 +367,9 @@ pub struct SandboxRef { /// Service URLs returned by sandbox creation, keyed by service name. The /// empty key identifies the unnamed service. Non-create reads leave this empty. pub service_urls: HashMap, + pub restart_count: u32, + pub next_restart_at_ms: Option, + pub main_process_started_at_ms: Option, } /// Reusable workload template revision used to create a sandbox. @@ -359,7 +383,6 @@ pub struct SandboxWorkloadTemplateProvenance { impl SandboxRef { pub(crate) fn from_proto(sandbox: proto::Sandbox) -> Self { let phase = sandbox.phase().into(); - let exit_code = sandbox.status.as_ref().and_then(|status| status.exit_code); let created_from_workload_template = sandbox .created_from_workload_template @@ -367,6 +390,23 @@ impl SandboxRef { name: p.name, resource_version: p.resource_version, }); + let (exit_code, restart_count, next_restart_at_ms, main_process_started_at_ms) = sandbox + .status + .as_ref() + .map_or((None, 0, None, None), |status| { + ( + status.exit_code, + status.restart_count, + status + .next_restart_time + .as_ref() + .and_then(|time| openshell_core::time::timestamp_to_millis(time).ok()), + status + .main_process_started_time + .as_ref() + .and_then(|time| openshell_core::time::timestamp_to_millis(time).ok()), + ) + }); let meta = sandbox.metadata.unwrap_or_default(); Self { id: meta.id, @@ -378,6 +418,9 @@ impl SandboxRef { exit_code, created_from_workload_template, service_urls: HashMap::new(), + restart_count, + next_restart_at_ms, + main_process_started_at_ms, } } } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 4a38361e1b..da0ebffef0 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -38,8 +38,8 @@ use openshell_core::proto::compute::v1::{ compute_driver_server::ComputeDriver, watch_sandboxes_event, }; use openshell_core::proto::{ - PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxSpec, SandboxStatus, - SandboxTemplate, SandboxWorkloadTemplate, ServiceEndpoint, SshSession, + PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxRestartPolicy, SandboxSpec, + SandboxStatus, SandboxTemplate, SandboxWorkloadTemplate, ServiceEndpoint, SshSession, }; use openshell_core::telemetry::TelemetryComputeDriver; use openshell_core::{ObjectLabels, ObjectWorkspace}; @@ -49,11 +49,11 @@ use std::fmt; use std::future::Future; use std::path::{Path, PathBuf}; use std::pin::Pin; -use std::sync::{Arc, Mutex as StdMutex, Weak}; +use std::sync::{Arc, Mutex as StdMutex, OnceLock, Weak}; use std::time::{Duration, Instant}; #[cfg(unix)] use tokio::net::UnixStream; -use tokio::sync::{Mutex, watch}; +use tokio::sync::{Mutex, Notify, watch}; use tonic::transport::Channel; #[cfg(unix)] use tonic::transport::Endpoint; @@ -319,6 +319,17 @@ async fn wait_for_startup( /// Interval between store-vs-backend reconciliation sweeps. const RECONCILE_INTERVAL: Duration = Duration::from_mins(1); +/// Restart intents live in sandbox rows. A lightweight holder-only scan makes +/// cross-replica writes and lost wakeups recover without changing the general +/// backend reconciliation cadence. +const RESTART_SCAN_INTERVAL: Duration = Duration::from_millis(500); +const RESTART_BACKOFF_BASE_DELAY_MS: i64 = 1_000; +const RESTART_MAX_DELAY_MS: i64 = 180_000; +const RESTART_STABILITY_WINDOW_MS: i64 = 10_000; +const RESTART_DRIVER_RETRY_DELAY_MS: i64 = 10_000; +const RESTART_TERMINAL_DELIVERY_GRACE_MS: i64 = 10_000; +const RESTART_READINESS_TIMEOUT_MS: i64 = 120_000; + /// How long a sandbox can remain provisioning in the store without a /// corresponding backend resource before it is considered orphaned. const ORPHAN_GRACE_PERIOD: Duration = Duration::from_mins(5); @@ -643,6 +654,9 @@ pub struct ComputeRuntime { /// clones: `ServerState` holds `ComputeRuntime` by value, so a per-clone /// table would make a token minted on one clone invisible to another. rootfs_tar_staging: Arc, + restart_authority: + Arc>>>, + restart_notify: Arc, } pub struct SandboxSyncGuard { @@ -738,6 +752,8 @@ impl ComputeRuntime { lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), replica_id: lease::replica_id(), rootfs_tar_staging, + restart_authority: Arc::new(OnceLock::new()), + restart_notify: Arc::new(Notify::new()), }) } @@ -1341,9 +1357,10 @@ impl ComputeRuntime { .map_err(Status::internal)?; return Ok(current); } - if !matches!(phase, SandboxPhase::Ready | SandboxPhase::Stopping) { + let automatic_restart = is_automatic_restart_transition(¤t); + if !matches!(phase, SandboxPhase::Ready | SandboxPhase::Stopping) && !automatic_restart { return Err(Status::failed_precondition(format!( - "sandbox must be Ready to stop (current phase: {phase:?})" + "sandbox must be Ready or in an automatic restart to stop (current phase: {phase:?})" ))); } @@ -2749,7 +2766,9 @@ impl ComputeRuntime { &self, shutdown_rx: watch::Receiver, startup_rx: watch::Receiver, + authority: Option>, ) { + let _ = self.restart_authority.set(authority); let runtime = Arc::new(self.clone()); if self.store.is_single_replica() { let deadline_runtime = runtime.clone(); @@ -2766,11 +2785,16 @@ impl ComputeRuntime { } Box::pin(watch_runtime.watch_loop(watch_shutdown)).await; }); + let reconcile_runtime = runtime.clone(); + let reconcile_shutdown = shutdown_rx.clone(); tokio::spawn(async move { - if !wait_for_startup(startup_rx, shutdown_rx.clone()).await { + if !wait_for_startup(startup_rx, reconcile_shutdown.clone()).await { return; } - runtime.reconcile_loop(shutdown_rx).await; + reconcile_runtime.reconcile_loop(reconcile_shutdown).await; + }); + tokio::spawn(async move { + runtime.restart_loop(shutdown_rx).await; }); } else { tokio::spawn(async move { @@ -2989,6 +3013,12 @@ impl ComputeRuntime { failed += 1; continue; } + if is_automatic_restart_transition(&sandbox) { + // The restart loop resumes both scheduled and claimed + // attempts. It always stops the old runtime before starting + // its replacement, including after a gateway crash. + continue; + } let sandbox_name = sandbox.object_name().to_string(); let generation_id = match sandbox_runtime_generation(&sandbox) { @@ -3470,6 +3500,12 @@ impl ComputeRuntime { runtime.reconcile_loop(cancel_rx).await; }); + let runtime = self.clone(); + let cancel_rx = cancel_tx.subscribe(); + let restart_handle = tokio::spawn(async move { + runtime.restart_loop(cancel_rx).await; + }); + loop { tokio::select! { () = tokio::time::sleep(LEASE_RENEWAL_INTERVAL) => { @@ -3497,6 +3533,7 @@ impl ComputeRuntime { let _ = watch_handle.await; let _ = reconcile_handle.await; let _ = deadline_handle.await; + let _ = restart_handle.await; return; } } @@ -3507,6 +3544,7 @@ impl ComputeRuntime { let _ = watch_handle.await; let _ = reconcile_handle.await; let _ = deadline_handle.await; + let _ = restart_handle.await; info!(replica = %lease.replica_id(), "reconciler lease lost — returning to standby"); } @@ -3568,126 +3606,554 @@ impl ComputeRuntime { } } - #[tracing::instrument( - name = "reconcile", - skip_all, - fields( - otel.name = "reconcile.sandboxes", - driver.name = %self.driver_info.name, - backend_count = tracing::field::Empty, - store_count = tracing::field::Empty, - ) - )] - async fn reconcile_store_with_backend(&self, grace_period: Duration) -> Result<(), String> { - let sweep_started_at_ms = openshell_core::time::now_ms(); - // Reclaims staging directories whose driver failed before its own - // cleanup ran, which the token table cannot see once consumed. - self.rootfs_tar_staging.sweep_orphans(); - let backend_sandboxes = self - .driver - .call( - openshell_otel::rpc::LIST_SANDBOXES, - None, - |driver| async move { - driver - .list_sandboxes(Request::new(ListSandboxesRequest {})) - .await - }, - ) - .await - .map_err(|e| e.to_string()) - .inspect_err(|_| crate::otel_tracing::mark_error(&tracing::Span::current()))? - .into_inner() - .sandboxes; - let backend_ids = backend_sandboxes - .iter() - .map(|sandbox| sandbox.id.clone()) - .collect::>(); - tracing::Span::current().record("backend_count", backend_sandboxes.len()); + async fn restart_loop(self: Arc, mut cancel: watch::Receiver) { + loop { + if let Err(err) = self.restart_due_sandboxes().await { + warn!(error = %err, "Sandbox restart sweep failed"); + } + tokio::select! { + () = tokio::time::sleep(RESTART_SCAN_INTERVAL) => {} + () = self.restart_notify.notified() => {} + _ = cancel.changed() => return, + } + } + } - for sandbox in backend_sandboxes { - self.reconcile_snapshot_sandbox(sandbox, sweep_started_at_ms) + async fn restart_due_sandboxes(&self) -> Result<(), String> { + let now_ms = openshell_core::time::now_ms(); + let mut offset = 0; + loop { + let records = self + .store + .list_by_type(Sandbox::object_type(), 1000, offset) .await - .inspect_err(|_| crate::otel_tracing::mark_error(&tracing::Span::current()))?; + .map_err(|err| err.to_string())?; + let page_len = records.len(); + for record in records { + let sandbox = match Sandbox::decode(record.payload.as_slice()) { + Ok(sandbox) => sandbox, + Err(err) => { + warn!(error = %err, "Failed to decode sandbox during restart sweep"); + continue; + } + }; + let phase = + SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); + let due = sandbox + .status + .as_ref() + .is_some_and(|status| restart_is_due(status, now_ms)); + if phase == SandboxPhase::Starting + && is_automatic_restart_transition(&sandbox) + && due + && let Err(err) = self.restart_sandbox_runtime(sandbox.object_id()).await + { + warn!( + sandbox_id = %sandbox.object_id(), + sandbox_name = %sandbox.object_name(), + error = %err, + "Automatic sandbox restart attempt failed" + ); + } + } + if page_len < 1000 { + break; + } + offset += u32::try_from(page_len).expect("restart scan page length is capped at 1000"); } + Ok(()) + } - let records = self + async fn restart_sandbox_runtime(&self, sandbox_id: &str) -> Result<(), String> { + let lifecycle_guard = self.lifecycle_gates.lock_for(sandbox_id).await; + let global_guard = self.lock_global_for_lifecycle(&lifecycle_guard).await; + let Some(current) = self .store - .collect_records(Sandbox::object_type(), ObjectListQuery::AllWorkspaces) + .get_message::(sandbox_id) .await - .map_err(|e| e.to_string()) - .inspect_err(|_| crate::otel_tracing::mark_error(&tracing::Span::current()))?; - tracing::Span::current().record("store_count", records.len()); + .map_err(|err| err.to_string())? + else { + return Ok(()); + }; + let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); + let now_ms = openshell_core::time::now_ms(); + let due = current + .status + .as_ref() + .is_some_and(|status| restart_is_due(status, now_ms)); + if phase != SandboxPhase::Starting || !is_automatic_restart_transition(¤t) || !due { + return Ok(()); + } - let grace_ms = grace_period.as_millis().try_into().unwrap_or(i64::MAX); + let sandbox_name = current.object_name().to_string(); + let authority = self.restart_authority.get().and_then(Option::as_deref); + let already_claimed = current + .status + .as_ref() + .is_some_and(|status| next_restart_at_ms(status) == 0); + let next_identity = if already_claimed { + None + } else { + authority + .map(|_| next_runtime_identity(¤t)) + .transpose() + .map_err(|status| status.to_string())? + }; + let claimed = if already_claimed { + current.clone() + } else { + self.store + .update_message_cas::( + sandbox_id, + sandbox_resource_version(¤t), + |sandbox| { + if let (Some(identity), Some(metadata)) = + (next_identity.as_ref(), sandbox.metadata.as_mut()) + { + identity.write(&mut metadata.annotations); + } + let status = sandbox.status.get_or_insert_with(Default::default); + // Zero durably claims this due attempt across gateway + // replicas. The readiness watchdog starts only after the + // old runtime has stopped. + set_next_restart_at_ms(status, 0); + upsert_ready_condition( + &mut sandbox.status, + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "SandboxRestarting".to_string(), + message: + "Replacing sandbox runtime after canonical main process exit" + .to_string(), + transition_time: None, + }, + ); + }, + ) + .await + .map_err(|err| err.to_string())? + }; + self.sandbox_index.update_from_sandbox(&claimed); + self.sandbox_watch_bus.notify(sandbox_id); + drop(global_guard); - for record in records { - let sandbox = match Sandbox::decode(record.payload.as_slice()) { - Ok(sandbox) => sandbox, - Err(err) => { - warn!(error = %err, "Failed to decode sandbox record during reconciliation"); - continue; - } - }; + let stop_result = self + .driver + .call( + openshell_otel::rpc::STOP_SANDBOX, + Some(sandbox_id), + |driver| { + let sandbox_id = sandbox_id.to_string(); + let sandbox_name = sandbox_name.clone(); + async move { + driver + .stop_sandbox(Request::new(StopSandboxRequest { + sandbox_id, + name: sandbox_name, + })) + .await + } + }, + ) + .await; + if let Err(status) = stop_result { + return self + .record_restart_driver_failure(&lifecycle_guard, sandbox_id, "stop", status) + .await; + } - if backend_ids.contains(sandbox.object_id()) { - continue; - } + self.cleanup_stopped_sandbox_sessions(&claimed).await?; - self.prune_missing_sandbox(record, sweep_started_at_ms, grace_ms) - .await - .inspect_err(|_| crate::otel_tracing::mark_error(&tracing::Span::current()))?; + let Some(armed) = self.arm_restart_readiness_watchdog(&claimed).await? else { + // A concurrent stop or delete changed durable intent while the + // old runtime was stopping. Do not recreate it. + return Ok(()); + }; + + let launch_authentication = serialize_persisted_launch_authentication(authority, &armed) + .map_err(|status| status.to_string())?; + let generation_id = sandbox_runtime_generation(&armed)?.into_string(); + let expected_runtime_identity = sandbox_compute_runtime_identity(¤t); + + let start_result = self + .driver + .call( + openshell_otel::rpc::START_SANDBOX, + Some(sandbox_id), + |driver| { + let sandbox_id = sandbox_id.to_string(); + let sandbox_name = sandbox_name.clone(); + async move { + driver + .start_sandbox(Request::new(StartSandboxRequest { + sandbox_id, + name: sandbox_name, + launch_authentication, + generation_id, + expected_runtime_identity, + })) + .await + } + }, + ) + .await; + let response = match start_result { + Ok(response) => response.into_inner(), + Err(status) => { + return self + .record_restart_driver_failure(&lifecycle_guard, sandbox_id, "start", status) + .await; + } + }; + if self.supports_sandbox_authentication() { + if response.runtime_identity.is_empty() { + return Err( + "compute driver returned an empty runtime identity after restart".into(), + ); + } + self.persist_runtime_binding( + sandbox_id, + &armed, + self.configured_driver_name(), + &response.runtime_identity, + &[SandboxPhase::Starting, SandboxPhase::Ready], + ) + .await?; } + self.enforce_lifecycle_after_restart_start(&armed).await?; + + info!( + sandbox_id, + sandbox_name, "Sandbox runtime restarted; waiting for replacement supervisor" + ); Ok(()) } - async fn apply_watch_event(&self, event: WatchSandboxesEvent) -> Result<(), String> { - let (operation, sandbox_id) = match &event.payload { - Some(watch_sandboxes_event::Payload::Sandbox(update)) => ( - "driver_watch.sandbox_updated", - update - .sandbox + /// Arm the replacement readiness watchdog immediately before starting + /// compute. The zero deadline written by the restart claim fences other + /// gateway replicas without charging slow driver shutdown time against + /// the replacement supervisor's readiness window. + async fn arm_restart_readiness_watchdog( + &self, + claimed: &Sandbox, + ) -> Result, String> { + let sandbox_id = claimed.object_id().to_string(); + let mut current = claimed.clone(); + for _ in 0..DELETE_PHASE_CAS_RETRY_LIMIT { + let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); + let claim_owned = phase == SandboxPhase::Starting + && is_automatic_restart_transition(¤t) + && current + .status .as_ref() - .map(|sandbox| sandbox.id.as_str()) - .unwrap_or_default(), - ), - Some(watch_sandboxes_event::Payload::Deleted(deleted)) => { - ("driver_watch.sandbox_deleted", deleted.sandbox_id.as_str()) + .is_some_and(|status| next_restart_at_ms(status) == 0); + if !claim_owned { + return Ok(None); } - Some(watch_sandboxes_event::Payload::PlatformEvent(platform_event)) => ( - "driver_watch.platform_event", - platform_event.sandbox_id.as_str(), - ), - None => return Ok(()), - }; - let span = tracing::info_span!( - "driver_watch", - otel.name = operation, - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - ); - async { - let result = self.apply_watch_event_inner(event).await; - if result.is_err() { - crate::otel_tracing::mark_error(&tracing::Span::current()); + + let readiness_deadline = + openshell_core::time::now_ms().saturating_add(RESTART_READINESS_TIMEOUT_MS); + match self + .store + .update_message_cas::( + &sandbox_id, + sandbox_resource_version(¤t), + |sandbox| { + let status = sandbox.status.get_or_insert_with(Default::default); + set_next_restart_at_ms(status, readiness_deadline); + }, + ) + .await + { + Ok(armed) => { + self.sandbox_index.update_from_sandbox(&armed); + self.sandbox_watch_bus.notify(&sandbox_id); + return Ok(Some(armed)); + } + Err(crate::persistence::PersistenceError::Conflict { .. }) => { + let Some(latest) = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|err| err.to_string())? + else { + return Ok(None); + }; + current = latest; + } + Err(err) => return Err(err.to_string()), } - result } - .instrument(span) - .await + + Ok(None) } - async fn apply_watch_event_inner(&self, event: WatchSandboxesEvent) -> Result<(), String> { - validate_driver_watch_event_timestamps(&event)?; - match event.payload { - Some(watch_sandboxes_event::Payload::Sandbox(sandbox)) => { - if let Some(sandbox) = sandbox.sandbox { - Box::pin(self.apply_sandbox_update(sandbox)).await?; - } - } - Some(watch_sandboxes_event::Payload::Deleted(deleted)) => { - self.apply_deleted(&deleted.sandbox_id).await?; + /// Reassert a concurrent stop/delete after the replacement start call. + /// The durable phase is authoritative across gateway replicas; the + /// compensating driver operation prevents a late start from leaving + /// running compute behind a stopped row or an orphan after deletion. + async fn enforce_lifecycle_after_restart_start(&self, armed: &Sandbox) -> Result<(), String> { + let sandbox_id = armed.object_id().to_string(); + let sandbox_name = armed.object_name().to_string(); + let latest = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|err| err.to_string())?; + let phase = latest.as_ref().map(|sandbox| { + SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) + }); + + match phase { + None | Some(SandboxPhase::Deleting) => { + let result = self + .driver + .call( + openshell_otel::rpc::DELETE_SANDBOX, + Some(&sandbox_id), + |driver| { + let sandbox_id = sandbox_id.clone(); + let sandbox_name = sandbox_name.clone(); + async move { + driver + .delete_sandbox(Request::new(DeleteSandboxRequest { + sandbox_id, + name: sandbox_name, + })) + .await + } + }, + ) + .await; + if let Err(status) = result + && status.code() != Code::NotFound + { + return Err(format!( + "failed to reassert concurrent sandbox delete after restart: {status}" + )); + } + } + Some(SandboxPhase::Stopping | SandboxPhase::Stopped) => { + let result = self + .driver + .call( + openshell_otel::rpc::STOP_SANDBOX, + Some(&sandbox_id), + |driver| { + let sandbox_id = sandbox_id.clone(); + let sandbox_name = sandbox_name.clone(); + async move { + driver + .stop_sandbox(Request::new(StopSandboxRequest { + sandbox_id, + name: sandbox_name, + })) + .await + } + }, + ) + .await; + if let Err(status) = result + && status.code() != Code::NotFound + { + return Err(format!( + "failed to reassert concurrent sandbox stop after restart: {status}" + )); + } + } + _ => {} + } + + Ok(()) + } + + async fn record_restart_driver_failure( + &self, + lifecycle_guard: &SandboxLifecycleGuard, + sandbox_id: &str, + operation: &str, + driver_status: Status, + ) -> Result<(), String> { + let _global_guard = self.lock_global_for_lifecycle(lifecycle_guard).await; + let Some(current) = self + .store + .get_message::(sandbox_id) + .await + .map_err(|err| err.to_string())? + else { + return Ok(()); + }; + if !is_automatic_restart_transition(¤t) { + return Ok(()); + } + let terminal = driver_status.code() == Code::NotFound; + let operation = operation.to_string(); + let driver_message = driver_status.message().to_string(); + let now_ms = openshell_core::time::now_ms(); + let updated = self + .store + .update_message_cas::( + sandbox_id, + sandbox_resource_version(¤t), + |sandbox| { + let status = sandbox.status.get_or_insert_with(Default::default); + if terminal { + set_next_restart_at_ms(status, 0); + sandbox.set_phase(SandboxPhase::Error as i32); + } else { + set_next_restart_at_ms( + status, + now_ms.saturating_add( + restart_delay_ms(status.restart_count.max(1)) + .max(RESTART_DRIVER_RETRY_DELAY_MS), + ), + ); + } + upsert_ready_condition( + &mut sandbox.status, + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "SandboxRestartFailed".to_string(), + message: format!( + "Sandbox restart {operation} failed: {driver_message}" + ), + transition_time: None, + }, + ); + }, + ) + .await + .map_err(|err| err.to_string())?; + self.sandbox_index.update_from_sandbox(&updated); + self.sandbox_watch_bus.notify(sandbox_id); + Err(format!( + "driver {operation} failed during sandbox restart: {driver_status}" + )) + } + + #[tracing::instrument( + name = "reconcile", + skip_all, + fields( + otel.name = "reconcile.sandboxes", + driver.name = %self.driver_info.name, + backend_count = tracing::field::Empty, + store_count = tracing::field::Empty, + ) + )] + async fn reconcile_store_with_backend(&self, grace_period: Duration) -> Result<(), String> { + let sweep_started_at_ms = openshell_core::time::now_ms(); + // Reclaims staging directories whose driver failed before its own + // cleanup ran, which the token table cannot see once consumed. + self.rootfs_tar_staging.sweep_orphans(); + let backend_sandboxes = self + .driver + .call( + openshell_otel::rpc::LIST_SANDBOXES, + None, + |driver| async move { + driver + .list_sandboxes(Request::new(ListSandboxesRequest {})) + .await + }, + ) + .await + .map_err(|e| e.to_string()) + .inspect_err(|_| crate::otel_tracing::mark_error(&tracing::Span::current()))? + .into_inner() + .sandboxes; + let backend_ids = backend_sandboxes + .iter() + .map(|sandbox| sandbox.id.clone()) + .collect::>(); + tracing::Span::current().record("backend_count", backend_sandboxes.len()); + + for sandbox in backend_sandboxes { + self.reconcile_snapshot_sandbox(sandbox, sweep_started_at_ms) + .await + .inspect_err(|_| crate::otel_tracing::mark_error(&tracing::Span::current()))?; + } + + let records = self + .store + .collect_records(Sandbox::object_type(), ObjectListQuery::AllWorkspaces) + .await + .map_err(|e| e.to_string()) + .inspect_err(|_| crate::otel_tracing::mark_error(&tracing::Span::current()))?; + tracing::Span::current().record("store_count", records.len()); + + let grace_ms = grace_period.as_millis().try_into().unwrap_or(i64::MAX); + + for record in records { + let sandbox = match Sandbox::decode(record.payload.as_slice()) { + Ok(sandbox) => sandbox, + Err(err) => { + warn!(error = %err, "Failed to decode sandbox record during reconciliation"); + continue; + } + }; + + if backend_ids.contains(sandbox.object_id()) { + continue; + } + + self.prune_missing_sandbox(record, sweep_started_at_ms, grace_ms) + .await + .inspect_err(|_| crate::otel_tracing::mark_error(&tracing::Span::current()))?; + } + + Ok(()) + } + + async fn apply_watch_event(&self, event: WatchSandboxesEvent) -> Result<(), String> { + let (operation, sandbox_id) = match &event.payload { + Some(watch_sandboxes_event::Payload::Sandbox(update)) => ( + "driver_watch.sandbox_updated", + update + .sandbox + .as_ref() + .map(|sandbox| sandbox.id.as_str()) + .unwrap_or_default(), + ), + Some(watch_sandboxes_event::Payload::Deleted(deleted)) => { + ("driver_watch.sandbox_deleted", deleted.sandbox_id.as_str()) + } + Some(watch_sandboxes_event::Payload::PlatformEvent(platform_event)) => ( + "driver_watch.platform_event", + platform_event.sandbox_id.as_str(), + ), + None => return Ok(()), + }; + let span = tracing::info_span!( + "driver_watch", + otel.name = operation, + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ); + async { + let result = self.apply_watch_event_inner(event).await; + if result.is_err() { + crate::otel_tracing::mark_error(&tracing::Span::current()); + } + result + } + .instrument(span) + .await + } + + async fn apply_watch_event_inner(&self, event: WatchSandboxesEvent) -> Result<(), String> { + validate_driver_watch_event_timestamps(&event)?; + match event.payload { + Some(watch_sandboxes_event::Payload::Sandbox(sandbox)) => { + if let Some(sandbox) = sandbox.sandbox { + Box::pin(self.apply_sandbox_update(sandbox)).await?; + } + } + Some(watch_sandboxes_event::Payload::Deleted(deleted)) => { + self.apply_deleted(&deleted.sandbox_id).await?; } Some(watch_sandboxes_event::Payload::PlatformEvent(platform_event)) => { if let Some(event) = platform_event.event { @@ -4007,6 +4473,15 @@ impl ComputeRuntime { if !connected && current_phase != SandboxPhase::Ready { return Ok(()); } + if connected + && is_automatic_restart_transition(¤t) + && current.status.as_ref().is_some_and(|status| { + !status.main_process_instance_id.is_empty() + && Some(status.main_process_instance_id.as_str()) == instance_id + }) + { + return Err("restarting sandbox requires a fresh supervisor instance".into()); + } let expected_resource_version = sandbox_resource_version(¤t); let result = self .store @@ -4017,9 +4492,18 @@ impl ComputeRuntime { if connected { ensure_supervisor_ready_status(&mut sandbox.status); let status = sandbox.status.get_or_insert_with(Default::default); + let same_instance = + Some(status.main_process_instance_id.as_str()) == instance_id; status.main_process_instance_id = instance_id.unwrap_or_default().to_string(); status.exit_code = None; + set_next_restart_at_ms(status, 0); + if !same_instance || main_process_started_at_ms(status) == 0 { + set_main_process_started_at_ms( + status, + openshell_core::time::now_ms(), + ); + } sandbox.set_phase(SandboxPhase::Ready as i32); } else { ensure_supervisor_not_ready_status(&mut sandbox.status); @@ -4109,6 +4593,11 @@ impl ComputeRuntime { ) { return Ok(()); } + let accepting_replacement_exit = existing.status.as_ref().is_some_and(|status| { + phase == SandboxPhase::Starting + && !status.main_process_instance_id.is_empty() + && status.main_process_instance_id != instance_id + }); if let Some(status) = existing.status.as_ref() { if !status.main_process_instance_id.is_empty() { // While Starting, the stored id belongs to the stopped @@ -4134,7 +4623,9 @@ impl ComputeRuntime { return Ok(()); } } - if let Some(current_exit_code) = status.exit_code { + if let Some(current_exit_code) = status.exit_code + && !accepting_replacement_exit + { if current_exit_code != exit_code { tracing::warn!( sandbox_id, @@ -4148,11 +4639,21 @@ impl ComputeRuntime { return Ok(()); } } + let restart = !has_specific_infrastructure_error(&existing) + && existing + .spec + .as_ref() + .is_some_and(|spec| should_restart_main_process(spec.restart_policy, exit_code)); + let now_ms = openshell_core::time::now_ms(); let expected_resource_version = sandbox_resource_version(&existing); let sandbox = self .store .update_message_cas::(sandbox_id, expected_resource_version, |sandbox| { - apply_main_process_exit(sandbox, instance_id, exit_code); + if restart { + apply_main_process_restart(sandbox, instance_id, exit_code, now_ms); + } else { + apply_main_process_exit(sandbox, instance_id, exit_code); + } }) .await .map_err(|error| error.to_string())?; @@ -4198,6 +4699,27 @@ impl ComputeRuntime { { return Err("main-process instance does not match the terminal result".to_string()); } + if is_terminal_delivery_pending(status) { + let restart_count = status.restart_count; + let delay_ms = restart_delay_ms(restart_count); + let updated = self + .store + .update_message_cas::( + sandbox_id, + sandbox_resource_version(&sandbox), + |sandbox| { + upsert_ready_condition( + &mut sandbox.status, + restart_ready_condition(restart_count, delay_ms), + ); + }, + ) + .await + .map_err(|error| error.to_string())?; + self.sandbox_index.update_from_sandbox(&updated); + self.sandbox_watch_bus.notify(sandbox_id); + self.restart_notify.notify_one(); + } Ok(()) } @@ -4772,12 +5294,7 @@ fn apply_main_process_exit(sandbox: &mut Sandbox, instance_id: &str, exit_code: // ContainerExited is only a provisional classification: replace it with // the canonical process result once its exit code is known. Preserve all // other infrastructure errors. - let preserve_infrastructure_error = sandbox.phase() == SandboxPhase::Error as i32 - && !sandbox.status.as_ref().is_some_and(|status| { - status.conditions.iter().any(|condition| { - condition.r#type == "Ready" && condition.reason == "ContainerExited" - }) - }); + let preserve_infrastructure_error = has_specific_infrastructure_error(sandbox); let status = sandbox.status.get_or_insert_with(SandboxStatus::default); status.main_process_instance_id = instance_id.to_string(); status.exit_code = Some(exit_code); @@ -4797,6 +5314,8 @@ fn apply_main_process_exit(sandbox: &mut Sandbox, instance_id: &str, exit_code: format!("Canonical main process exited with status {exit_code}"), ) }; + set_next_restart_at_ms(status, 0); + set_main_process_started_at_ms(status, 0); upsert_ready_condition( &mut sandbox.status, SandboxCondition { @@ -4810,6 +5329,15 @@ fn apply_main_process_exit(sandbox: &mut Sandbox, instance_id: &str, exit_code: sandbox.set_phase(phase as i32); } +fn has_specific_infrastructure_error(sandbox: &Sandbox) -> bool { + sandbox.phase() == SandboxPhase::Error as i32 + && !sandbox.status.as_ref().is_some_and(|status| { + status.conditions.iter().any(|condition| { + condition.r#type == "Ready" && condition.reason == "ContainerExited" + }) + }) +} + fn is_failed_main_process_result(sandbox: &Sandbox) -> bool { sandbox.phase() == SandboxPhase::Error as i32 && sandbox.status.as_ref().is_some_and(|status| { @@ -4822,11 +5350,156 @@ fn is_failed_main_process_result(sandbox: &Sandbox) -> bool { }) } -/// Connect to an unmanaged remote compute driver that is already listening on -/// `socket_path` and return the acquired endpoint. -/// -/// The gateway does not spawn or own the driver process — the operator is -/// responsible for placing the driver alongside the gateway and granting the +fn should_restart_main_process(policy: i32, exit_code: i32) -> bool { + match SandboxRestartPolicy::try_from(policy).unwrap_or(SandboxRestartPolicy::Never) { + SandboxRestartPolicy::Always => true, + SandboxRestartPolicy::OnFailure => exit_code != 0, + SandboxRestartPolicy::Unspecified | SandboxRestartPolicy::Never => false, + } +} + +fn is_automatic_restart_status(status: &SandboxStatus) -> bool { + status.restart_count > 0 && status.exit_code.is_some() +} + +fn is_automatic_restart_transition(sandbox: &Sandbox) -> bool { + SandboxPhase::try_from(sandbox.phase()) == Ok(SandboxPhase::Starting) + && sandbox + .status + .as_ref() + .is_some_and(is_automatic_restart_status) +} + +fn restart_delay_ms(restart_count: u32) -> i64 { + if restart_count <= 1 { + return 0; + } + let shift = restart_count.saturating_sub(2).min(31); + RESTART_BACKOFF_BASE_DELAY_MS + .saturating_mul(1_i64.checked_shl(shift).unwrap_or(i64::MAX)) + .min(RESTART_MAX_DELAY_MS) +} + +fn is_terminal_delivery_pending(status: &SandboxStatus) -> bool { + status.conditions.iter().any(|condition| { + condition.r#type == "Ready" && condition.reason == "MainProcessExitDraining" + }) +} + +fn restart_is_due(status: &SandboxStatus, now_ms: i64) -> bool { + let deadline_ms = next_restart_at_ms(status); + if deadline_ms > now_ms { + return false; + } + if !is_terminal_delivery_pending(status) { + return true; + } + let exit_time_ms = deadline_ms.saturating_sub(restart_delay_ms(status.restart_count)); + now_ms >= exit_time_ms.saturating_add(RESTART_TERMINAL_DELIVERY_GRACE_MS) +} + +fn restart_ready_condition(restart_count: u32, delay_ms: i64) -> SandboxCondition { + let (reason, message) = if delay_ms == 0 { + ( + "MainProcessRestartScheduled", + format!("Canonical main process exited; restart {restart_count} scheduled immediately"), + ) + } else { + let delay_seconds = delay_ms / 1_000; + let delay_unit = if delay_seconds == 1 { + "second" + } else { + "seconds" + }; + ( + "MainProcessRestartBackoff", + format!( + "Canonical main process exited; restart {restart_count} scheduled in {delay_seconds} {delay_unit}" + ), + ) + }; + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: reason.to_string(), + message, + transition_time: None, + } +} + +fn next_restart_at_ms(status: &SandboxStatus) -> i64 { + status + .next_restart_time + .as_ref() + .and_then(|time| openshell_core::time::timestamp_to_millis(time).ok()) + .unwrap_or_default() +} + +fn set_next_restart_at_ms(status: &mut SandboxStatus, millis: i64) { + status.next_restart_time = (millis > 0) + .then(|| openshell_core::time::timestamp_from_millis(millis).ok()) + .flatten(); +} + +fn main_process_started_at_ms(status: &SandboxStatus) -> i64 { + status + .main_process_started_time + .as_ref() + .and_then(|time| openshell_core::time::timestamp_to_millis(time).ok()) + .unwrap_or_default() +} + +fn set_main_process_started_at_ms(status: &mut SandboxStatus, millis: i64) { + status.main_process_started_time = (millis > 0) + .then(|| openshell_core::time::timestamp_from_millis(millis).ok()) + .flatten(); +} + +#[cfg(test)] +fn restart_timestamp(millis: i64) -> Option { + (millis > 0) + .then(|| openshell_core::time::timestamp_from_millis(millis).ok()) + .flatten() +} + +fn apply_main_process_restart( + sandbox: &mut Sandbox, + instance_id: &str, + exit_code: i32, + now_ms: i64, +) { + let status = sandbox.status.get_or_insert_with(SandboxStatus::default); + let stable_run = main_process_started_at_ms(status) > 0 + && now_ms.saturating_sub(main_process_started_at_ms(status)) >= RESTART_STABILITY_WINDOW_MS; + status.restart_count = if stable_run || status.restart_count == 0 { + 1 + } else { + status.restart_count.saturating_add(1) + }; + let delay_ms = restart_delay_ms(status.restart_count); + status.main_process_instance_id = instance_id.to_string(); + status.exit_code = Some(exit_code); + set_main_process_started_at_ms(status, 0); + set_next_restart_at_ms(status, now_ms.saturating_add(delay_ms)); + upsert_ready_condition( + &mut sandbox.status, + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "MainProcessExitDraining".to_string(), + message: "Canonical main process exited; waiting for terminal delivery before restart" + .to_string(), + transition_time: None, + }, + ); + sandbox.set_phase(SandboxPhase::Starting as i32); +} + +/// Connect to an unmanaged remote compute driver that is already listening on +/// `socket_path` and return the acquired endpoint. +/// +/// The gateway does not spawn or own the driver process — the operator is +/// responsible for placing the driver alongside the gateway and granting the /// gateway uid read/write on the socket. The host portion of the URL is /// ignored because the connector resolves to the UDS rather than DNS. #[cfg(unix)] @@ -5300,6 +5973,9 @@ fn public_status_from_driver( configuration_admission: None, configuration_activated: None, provisioning: None, + restart_count: 0, + next_restart_time: None, + main_process_started_time: None, } } @@ -5365,7 +6041,24 @@ fn apply_driver_snapshot( }, ); + // Driver status has no canonical-main fields. Preserve the gateway-owned + // process generation and restart controller state across every snapshot. + if let (Some(old_status), Some(new_status)) = (sandbox.status.as_ref(), status.as_mut()) { + new_status + .main_process_instance_id + .clone_from(&old_status.main_process_instance_id); + new_status.exit_code = old_status.exit_code; + new_status.restart_count = old_status.restart_count; + new_status.next_restart_time = old_status.next_restart_time; + new_status.main_process_started_time = old_status.main_process_started_time; + } + phase = match old_phase { + // Backoff and replacement are gateway-owned. Driver snapshots can + // describe the old runtime stopping but cannot cancel restart intent. + SandboxPhase::Starting if is_automatic_restart_transition(sandbox) => { + SandboxPhase::Starting + } // SIGTERM-driven runtime exits are reported as a runtime restart by // Docker and Podman. While an explicit stop owns this durable // transition, preserve Stopping so the stop result chooses whether @@ -5776,9 +6469,9 @@ fn rewrite_user_facing_conditions(status: &mut Option, spec: Opti } /// Phases for which a sandbox should have a running compute resource. -/// `Deleting` and `Error` are intentionally excluded: deletion is in -/// progress, or the sandbox has already failed and should not be -/// silently revived. `Unspecified` is included because it is the proto +/// Terminal and lifecycle-controller-owned phases are intentionally excluded: +/// they must not be silently revived or bypass a persisted transition/backoff. +/// `Unspecified` is included because it is the proto /// default value; persisted rows with that value should be reconciled /// from the live driver state rather than skipped forever. fn sandbox_phase_should_be_running(phase: SandboxPhase) -> bool { @@ -5815,6 +6508,9 @@ fn apply_lifecycle_phase(sandbox: &mut Sandbox, phase: SandboxPhase, reason: &st // Retain the previous instance id as a tombstone until the restarted // supervisor registers its new id. status.exit_code = None; + status.restart_count = 0; + set_next_restart_at_ms(status, 0); + set_main_process_started_at_ms(status, 0); if phase == SandboxPhase::Starting { status.provisioning = Some(provisioning_deadline::new_record( openshell_core::time::now_ms(), @@ -6201,6 +6897,8 @@ pub fn new_test_runtime_with_driver( lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), replica_id: "test-replica".to_string(), rootfs_tar_staging: Arc::new(rootfs_tar::RootfsTarStagingRegistry::disabled()), + restart_authority: Arc::new(OnceLock::new()), + restart_notify: Arc::new(Notify::new()), } } @@ -7248,6 +7946,8 @@ mod tests { lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), replica_id: "test-replica".to_string(), rootfs_tar_staging: Arc::new(rootfs_tar::RootfsTarStagingRegistry::disabled()), + restart_authority: Arc::new(OnceLock::new()), + restart_notify: Arc::new(Notify::new()), } } @@ -7893,6 +8593,7 @@ mod tests { #[test] fn main_process_exit_preserves_specific_infrastructure_error() { let mut sandbox = error_sandbox_record("sb-1", "sandbox-a", "BackendResourceMissing"); + assert!(has_specific_infrastructure_error(&sandbox)); apply_main_process_exit(&mut sandbox, "instance-1", 0); assert_eq!( @@ -7907,6 +8608,595 @@ mod tests { })); } + #[test] + fn restart_policy_matches_kubernetes_exit_semantics() { + assert!(!should_restart_main_process( + SandboxRestartPolicy::Never as i32, + 1 + )); + assert!(!should_restart_main_process( + SandboxRestartPolicy::OnFailure as i32, + 0 + )); + assert!(should_restart_main_process( + SandboxRestartPolicy::OnFailure as i32, + 1 + )); + assert!(should_restart_main_process( + SandboxRestartPolicy::Always as i32, + 0 + )); + } + + #[test] + fn restart_backoff_starts_after_first_exit_and_caps_at_three_minutes() { + assert_eq!(restart_delay_ms(0), 0); + assert_eq!(restart_delay_ms(1), 0); + assert_eq!(restart_delay_ms(2), 1_000); + assert_eq!(restart_delay_ms(3), 2_000); + assert_eq!(restart_delay_ms(4), 4_000); + assert_eq!(restart_delay_ms(8), 64_000); + assert_eq!(restart_delay_ms(9), 128_000); + assert_eq!(restart_delay_ms(10), 180_000); + assert_eq!(restart_delay_ms(20), 180_000); + } + + #[test] + fn unfinished_terminal_delivery_gets_bounded_restart_fallback() { + let now_ms = openshell_core::time::now_ms(); + let status = SandboxStatus { + restart_count: 2, + next_restart_time: restart_timestamp(now_ms + restart_delay_ms(2)), + conditions: vec![SandboxCondition { + r#type: "Ready".into(), + reason: "MainProcessExitDraining".into(), + ..Default::default() + }], + ..Default::default() + }; + + assert!(!restart_is_due(&status, now_ms + restart_delay_ms(2))); + assert!(!restart_is_due( + &status, + now_ms + RESTART_TERMINAL_DELIVERY_GRACE_MS - 1 + )); + assert!(restart_is_due( + &status, + now_ms + RESTART_TERMINAL_DELIVERY_GRACE_MS + )); + } + + #[tokio::test] + async fn always_policy_schedules_zero_exit_once_and_new_supervisor_becomes_ready() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready); + sandbox.spec = Some(SandboxSpec { + restart_policy: SandboxRestartPolicy::Always as i32, + ..Default::default() + }); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Ready as i32, + main_process_instance_id: "instance-1".into(), + main_process_started_time: restart_timestamp(openshell_core::time::now_ms()), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + let before_exit_ms = openshell_core::time::now_ms(); + runtime + .main_process_exited("sb-1", "instance-1", 0) + .await + .unwrap(); + runtime + .main_process_exited("sb-1", "instance-1", 0) + .await + .unwrap(); + + let restarting = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(restarting.phase(), SandboxPhase::Starting as i32); + let status = restarting.status.unwrap(); + assert_eq!(status.restart_count, 1); + assert_eq!(status.exit_code, Some(0)); + assert!(next_restart_at_ms(&status) >= before_exit_ms); + assert!(next_restart_at_ms(&status) <= openshell_core::time::now_ms()); + assert!(!restart_is_due(&status, openshell_core::time::now_ms())); + assert!(status.conditions.iter().any(|condition| { + condition.r#type == "Ready" && condition.reason == "MainProcessExitDraining" + })); + + runtime + .finalize_main_process_exit("sb-1", "instance-1") + .await + .unwrap(); + let finalized = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + let status = finalized.status.unwrap(); + assert!(restart_is_due(&status, openshell_core::time::now_ms())); + assert_eq!( + status + .conditions + .iter() + .find(|condition| condition.r#type == "Ready") + .map(|condition| condition.message.as_str()), + Some("Canonical main process exited; restart 1 scheduled immediately") + ); + assert!(status.conditions.iter().any(|condition| { + condition.r#type == "Ready" && condition.reason == "MainProcessRestartScheduled" + })); + tokio::time::timeout( + Duration::from_millis(100), + runtime.restart_notify.notified(), + ) + .await + .expect("finalized restart should wake the local restart worker"); + + runtime + .supervisor_session_connected("sb-1", "instance-2") + .await + .unwrap(); + let ready = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(ready.phase(), SandboxPhase::Ready as i32); + let status = ready.status.unwrap(); + assert_eq!(status.main_process_instance_id, "instance-2"); + assert_eq!(status.exit_code, None); + assert_eq!(next_restart_at_ms(&status), 0); + assert!(main_process_started_at_ms(&status) > 0); + } + + #[tokio::test] + async fn replacement_exit_before_supervisor_connect_uses_new_instance_result() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.spec = Some(SandboxSpec { + restart_policy: SandboxRestartPolicy::OnFailure as i32, + ..Default::default() + }); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + main_process_instance_id: "instance-1".into(), + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() + 120_000), + conditions: vec![SandboxCondition { + r#type: "Ready".into(), + status: "False".into(), + reason: "SandboxRestarting".into(), + ..Default::default() + }], + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime + .main_process_exited("sb-1", "instance-2", 0) + .await + .unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Completed as i32); + let status = stored.status.unwrap(); + assert_eq!(status.main_process_instance_id, "instance-2"); + assert_eq!(status.exit_code, Some(0)); + assert_eq!(next_restart_at_ms(&status), 0); + } + + #[tokio::test] + async fn restarting_sandbox_rejects_previous_supervisor_instance() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + main_process_instance_id: "instance-1".into(), + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() + 10_000), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + let error = runtime + .supervisor_session_connected("sb-1", "instance-1") + .await + .unwrap_err(); + + assert!(error.contains("requires a fresh supervisor instance")); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Starting as i32); + assert_eq!( + stored.status.unwrap().main_process_instance_id, + "instance-1" + ); + } + + #[tokio::test] + async fn same_main_reconnect_preserves_process_start_time() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Ready as i32, + main_process_instance_id: "instance-1".into(), + main_process_started_time: restart_timestamp(12_345), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime + .supervisor_session_connected("sb-1", "instance-1") + .await + .unwrap(); + + let reconnected = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + main_process_started_at_ms(&reconnected.status.unwrap()), + 12_345 + ); + } + + #[tokio::test] + async fn on_failure_policy_keeps_zero_exit_terminal() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready); + sandbox.spec = Some(SandboxSpec { + restart_policy: SandboxRestartPolicy::OnFailure as i32, + ..Default::default() + }); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Ready as i32, + main_process_instance_id: "instance-1".into(), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime + .main_process_exited("sb-1", "instance-1", 0) + .await + .unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Completed as i32); + assert_eq!(stored.status.unwrap().restart_count, 0); + } + + #[test] + fn stable_main_run_resets_restart_backoff_count() { + let now_ms = openshell_core::time::now_ms(); + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready); + sandbox.status = Some(SandboxStatus { + restart_count: 5, + main_process_started_time: restart_timestamp(now_ms - RESTART_STABILITY_WINDOW_MS), + ..Default::default() + }); + + apply_main_process_restart(&mut sandbox, "instance-6", 9, now_ms); + + let status = sandbox.status.unwrap(); + assert_eq!(status.restart_count, 1); + assert_eq!(next_restart_at_ms(&status), now_ms); + } + + #[test] + fn short_main_run_keeps_restart_backoff_count() { + let now_ms = openshell_core::time::now_ms(); + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready); + sandbox.status = Some(SandboxStatus { + restart_count: 5, + main_process_started_time: restart_timestamp(now_ms - RESTART_STABILITY_WINDOW_MS + 1), + ..Default::default() + }); + + apply_main_process_restart(&mut sandbox, "instance-6", 9, now_ms); + + let status = sandbox.status.unwrap(); + assert_eq!(status.restart_count, 6); + assert_eq!(next_restart_at_ms(&status), now_ms + 16_000); + } + + #[tokio::test] + async fn due_restart_stops_and_starts_driver_without_exposing_stopped_phase() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.spec = Some(SandboxSpec { + restart_policy: SandboxRestartPolicy::OnFailure as i32, + ..Default::default() + }); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + main_process_instance_id: "instance-1".into(), + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime.restart_sandbox_runtime("sb-1").await.unwrap(); + + assert_eq!(driver.stop_calls(), 1); + assert_eq!(driver.start_calls(), 1); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Starting as i32); + let status = stored.status.unwrap(); + assert_eq!(status.exit_code, Some(9)); + assert!(next_restart_at_ms(&status) > openshell_core::time::now_ms()); + assert!(status.conditions.iter().any(|condition| { + condition.r#type == "Ready" && condition.reason == "SandboxRestarting" + })); + } + + #[tokio::test] + async fn restart_readiness_deadline_starts_after_driver_stop() { + let driver = ControlledDriver::new(); + driver.block_stop(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + let restart_runtime = runtime.clone(); + let restart = + tokio::spawn(async move { restart_runtime.restart_sandbox_runtime("sb-1").await }); + driver.stop_started.notified().await; + + let claimed = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(next_restart_at_ms(&claimed.status.unwrap()), 0); + + driver.release_stop(); + restart.await.unwrap().unwrap(); + let started = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert!(next_restart_at_ms(&started.status.unwrap()) > openshell_core::time::now_ms()); + } + + #[tokio::test] + async fn claimed_restart_replays_stop_before_start_after_gateway_recovery() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + main_process_instance_id: "previous-instance".into(), + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(0), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime.restart_due_sandboxes().await.unwrap(); + + assert_eq!(driver.stop_calls(), 1); + assert_eq!(driver.start_calls(), 1); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert!(next_restart_at_ms(&stored.status.unwrap()) > openshell_core::time::now_ms()); + } + + #[tokio::test] + async fn concurrent_stop_transition_prevents_replacement_start() { + let driver = ControlledDriver::new(); + driver.block_stop(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + let restart_runtime = runtime.clone(); + let restart = + tokio::spawn(async move { restart_runtime.restart_sandbox_runtime("sb-1").await }); + driver.stop_started.notified().await; + let claimed = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + runtime + .store + .update_message_cas::( + "sb-1", + sandbox_resource_version(&claimed), + |sandbox| sandbox.set_phase(SandboxPhase::Stopping as i32), + ) + .await + .unwrap(); + + driver.release_stop(); + restart.await.unwrap().unwrap(); + assert_eq!(driver.start_calls(), 0); + } + + #[tokio::test] + async fn concurrent_delete_after_replacement_start_is_reasserted() { + let driver = ControlledDriver::new(); + driver.block_start(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + let restart_runtime = runtime.clone(); + let restart = + tokio::spawn(async move { restart_runtime.restart_sandbox_runtime("sb-1").await }); + driver.start_started.notified().await; + let armed = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + runtime + .store + .update_message_cas::("sb-1", sandbox_resource_version(&armed), |sandbox| { + sandbox.set_phase(SandboxPhase::Deleting as i32); + }) + .await + .unwrap(); + + driver.release_start(); + restart.await.unwrap().unwrap(); + assert_eq!(driver.delete_calls(), 1); + } + + #[tokio::test] + async fn retryable_restart_driver_failure_reschedules_attempt() { + let driver = ControlledDriver::new(); + driver.set_stop_outcome(ControlledLifecycleOutcome::Error("transport unavailable")); + let runtime = test_runtime(driver).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + let before_failure_ms = openshell_core::time::now_ms(); + let error = runtime.restart_sandbox_runtime("sb-1").await.unwrap_err(); + + assert!(error.contains("transport unavailable")); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Starting as i32); + let status = stored.status.unwrap(); + assert!( + next_restart_at_ms(&status) + >= before_failure_ms.saturating_add(RESTART_DRIVER_RETRY_DELAY_MS) + ); + assert!(status.conditions.iter().any(|condition| { + condition.r#type == "Ready" && condition.reason == "SandboxRestartFailed" + })); + } + + #[tokio::test] + async fn missing_runtime_during_restart_is_terminal() { + let driver = ControlledDriver::new(); + driver.set_stop_outcome(ControlledLifecycleOutcome::NotFound); + let runtime = test_runtime(driver).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime.restart_sandbox_runtime("sb-1").await.unwrap_err(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Error as i32); + let status = stored.status.unwrap(); + assert_eq!(next_restart_at_ms(&status), 0); + assert_eq!(status.exit_code, Some(9)); + } + + #[tokio::test] + async fn manual_stop_cancels_pending_restart() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + exit_code: Some(9), + restart_count: 2, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() + 60_000), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + let stopped = runtime.stop_sandbox("default", "sandbox-a").await.unwrap(); + + assert_eq!(stopped.phase(), SandboxPhase::Stopped as i32); + let status = stopped.status.unwrap(); + assert_eq!(status.exit_code, None); + assert_eq!(status.restart_count, 0); + assert_eq!(next_restart_at_ms(&status), 0); + assert_eq!(driver.stop_calls(), 1); + } + #[tokio::test] async fn stale_main_process_exit_is_acknowledged_without_replacing_active_instance() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index b4544266e7..8e3aa58372 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -40,7 +40,7 @@ use openshell_core::proto::{ }; use openshell_core::proto::{ BeginRootfsTarStagingRequest, BeginRootfsTarStagingResponse, Sandbox, SandboxPhase, - SandboxTemplate, SshSession, + SandboxRestartPolicy, SandboxTemplate, SshSession, }; use openshell_core::telemetry::{ LifecycleOperation, LifecycleResource, SandboxTemplateSource, TelemetryOutcome, @@ -238,6 +238,13 @@ pub(super) async fn resolve_and_authorize_sandbox_name( Ok(sandbox) } +fn require_ready_sandbox(sandbox: &Sandbox) -> Result<(), Status> { + match SandboxPhase::try_from(sandbox.phase()).ok() { + Some(SandboxPhase::Ready) => Ok(()), + _ => Err(Status::failed_precondition("sandbox is not ready")), + } +} + fn generate_routable_name() -> String { let name = petname::petname(2, "-").unwrap_or_else(generate_name); let mut truncated = &name[..name.len().min(MAX_ROUTABLE_NAME_LEN)]; @@ -420,6 +427,13 @@ async fn handle_create_sandbox_inner( validate_create_sandbox_request_pre_io(&request, &workload_template_name)?; + // Validate labels (keys and values must meet Kubernetes requirements). + for (key, value) in &request.labels { + crate::grpc::validation::validate_label_key(key)?; + crate::grpc::validation::validate_label_value(value)?; + } + crate::grpc::validation::validate_annotations(&request.annotations, "annotations")?; + let authz = authorize_workspace( &state.store, &state.admin_role, @@ -454,12 +468,16 @@ async fn handle_create_sandbox_inner( resolved.providers = governance_spec.providers; resolved.command = governance_spec.command; resolved.tty = governance_spec.tty; + resolved.restart_policy = governance_spec.restart_policy; (resolved, Some(provenance)) }; // Attachment identity belongs to the gateway. Accepting an epoch from a // create request or workload template could revive stale installation proof. spec.provider_attachment_epoch = uuid::Uuid::new_v4().to_string(); + if spec.restart_policy == SandboxRestartPolicy::Unspecified as i32 { + spec.restart_policy = SandboxRestartPolicy::Never as i32; + } // Leave an omitted command empty rather than persisting a concrete shell: // the sandbox boundary resolves the default login shell against the agent image @@ -2416,9 +2434,7 @@ pub(super) async fn handle_exec_sandbox( completion.ensure_target(sandbox.object_id())?; } - if SandboxPhase::try_from(sandbox.phase()).ok() != Some(SandboxPhase::Ready) { - return Err(Status::failed_precondition("sandbox is not ready")); - } + require_ready_sandbox(&sandbox)?; // Open a relay channel through the supervisor session. Use a 15s // session-wait timeout, enough to cover a transient supervisor reconnect @@ -2931,9 +2947,7 @@ pub(super) async fn handle_exec_sandbox_interactive_start( completion.ensure_target(sandbox.object_id())?; } - if SandboxPhase::try_from(sandbox.phase()).ok() != Some(SandboxPhase::Ready) { - return Err(Status::failed_precondition("sandbox is not ready")); - } + require_ready_sandbox(&sandbox)?; let (channel_id, relay_rx) = crate::supervisor_session::open_routed_relay_with_target( state, @@ -6657,6 +6671,10 @@ mod tests { .into_inner(); let created = response.sandbox.expect("created sandbox"); + assert_eq!( + created.spec.as_ref().unwrap().restart_policy(), + SandboxRestartPolicy::Never + ); assert_eq!( created .metadata @@ -7549,7 +7567,7 @@ mod tests { fn template_create_sandbox_spec_field_policy_is_exhaustive() { assert_proto_fields_classified( "openshell.v1.SandboxSpec", - &["policy", "providers", "command", "tty"], + &["policy", "providers", "command", "tty", "restart_policy"], &[ "log_level", "environment", diff --git a/crates/openshell-server/src/grpc/validation.rs b/crates/openshell-server/src/grpc/validation.rs index 4218f296da..f72c571ba5 100644 --- a/crates/openshell-server/src/grpc/validation.rs +++ b/crates/openshell-server/src/grpc/validation.rs @@ -10,7 +10,7 @@ use openshell_core::proto::{ CredentialHandle, ExecSandboxRequest, Provider, SandboxPolicy as ProtoSandboxPolicy, - SandboxSpec, SandboxTemplate, + SandboxRestartPolicy, SandboxSpec, SandboxTemplate, }; use openshell_core::rpc_error::invalid_argument; use prost::Message; @@ -206,6 +206,7 @@ pub(super) fn validate_sandbox_spec(name: &str, spec: &SandboxSpec) -> Result<() if !spec.command.is_empty() { validate_main_process_command(&spec.command)?; } + validate_restart_policy(spec)?; // --- spec.policy serialized size --- validate_sandbox_policy_size(spec)?; @@ -222,10 +223,21 @@ pub(super) fn validate_sandbox_governance_spec( if !spec.command.is_empty() { validate_main_process_command(&spec.command)?; } + validate_restart_policy(spec)?; validate_sandbox_policy_size(spec)?; Ok(()) } +fn validate_restart_policy(spec: &SandboxSpec) -> Result<(), Status> { + SandboxRestartPolicy::try_from(spec.restart_policy).map_err(|_| { + invalid_argument( + "spec.restart_policy", + format!("unknown restart_policy value: {}", spec.restart_policy), + ) + })?; + Ok(()) +} + fn validate_sandbox_name(name: &str) -> Result<(), Status> { if !name.is_empty() && name.len() > MAX_ROUTABLE_NAME_LEN { return Err(invalid_argument( @@ -1227,6 +1239,17 @@ mod tests { assert!(validate_sandbox_spec("", &default_spec()).is_ok()); } + #[test] + fn validate_sandbox_spec_rejects_unknown_restart_policy() { + let spec = SandboxSpec { + restart_policy: 99, + ..Default::default() + }; + let err = validate_sandbox_spec("", &spec).unwrap_err(); + assert_eq!(err.code(), Code::InvalidArgument); + assert!(err.message().contains("restart_policy")); + } + #[test] fn validate_sandbox_spec_accepts_exact_main_process_argv() { let spec = SandboxSpec { diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index ea1f74c0f4..01d710a178 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -949,9 +949,11 @@ pub(crate) async fn run_server( // Deadlines must run while restored supervisors wait for policy repair. let (startup_tx, startup_rx) = watch::channel(false); - state - .compute - .spawn_watchers(shutdown_rx.clone(), startup_rx); + state.compute.spawn_watchers( + shutdown_rx.clone(), + startup_rx, + state.sandbox_session_jwt_authority.clone(), + ); // Serve the gateway before reconciling persisted sandboxes so restored // supervisors can fetch policy and register their sessions. diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 16a733415c..f9283b8cca 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -118,16 +118,15 @@ mod tests { const STORAGE_V1_SCHEMA_SHA256: &str = "d68401809d8cea445c35233ef32412bbd041cb2ac5acaf368a0d0bf74d2ddf17"; - // Carries this branch's exec request IDs together with main's opaque watch - // cursor and well-known time types. These unreleased public-only fields add - // no messages or enums and touch no stored type, so the durable and overlap - // fingerprints below remain unchanged. + // Restart policy is stored in SandboxSpec, and the count and well-known + // timestamps are stored in SandboxStatus. Legacy payloads decode with + // Unspecified (treated as Never), zero count, and absent timestamps. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "8fb59b0932ec2f227fdec2d46b6204925e79595695810a247bef731ddd632594"; + "b1f9b34f035234e1032685eb4fc829950acad63a971b3abd37a2ccbbec783531"; const DURABLE_SCHEMA_SHA256: &str = - "9eeaa29dfba187bff69fb7bc4f9a13a0f1d7be3f7049a38c8f0e20ce77ec7d8b"; + "517561b578c88d28ffd74d128faf668e65aef03c784dd794aeb5208e1dbf6de3"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = - "a6e97fdde30c439ffaa03c2952a43033f8ea338fed6b1456ebe2d7d8af14e834"; + "d60c0a91163bcdd6c29e24c64e0f00555f914240465f94295d800e9063171bef"; // A persisted Sandbox without endpoint status retains its lifecycle fields; // the absent repeated field decodes empty and needs no database rewrite. const SANDBOX_WITHOUT_ENDPOINT_STATUS: &str = "0a1e0a0a73616e64626f782d6964120773616e64626f783a0764656661756c741a2b0a0773616e64626f782a0d0a05526561647912045472756530023807420d73757065727669736f722d6964"; @@ -590,9 +589,9 @@ mod tests { overlap_hash.as_str(), ), ( - (304, 25), - (92, 19), - (80, 19), + (304, 26), + (92, 20), + (80, 20), PUBLIC_RPC_SCHEMA_SHA256, DURABLE_SCHEMA_SHA256, PUBLIC_DURABLE_OVERLAP_SHA256 @@ -651,6 +650,9 @@ mod tests { assert_eq!(status.current_policy_version, 7); assert_eq!(status.main_process_instance_id, "supervisor-id"); assert_eq!(status.exit_code, None); + assert_eq!(status.restart_count, 0); + assert!(status.next_restart_time.is_none()); + assert!(status.main_process_started_time.is_none()); assert_eq!(status.conditions.len(), 1); assert_eq!(status.conditions[0].r#type, "Ready"); assert_eq!(status.conditions[0].status, "True"); diff --git a/crates/openshell-tui/src/app.rs b/crates/openshell-tui/src/app.rs index 7a53343142..08ab650386 100644 --- a/crates/openshell-tui/src/app.rs +++ b/crates/openshell-tui/src/app.rs @@ -656,6 +656,10 @@ pub struct App { pub sandbox_ages: Vec, pub sandbox_created: Vec, pub sandbox_images: Vec, + pub sandbox_restart_policies: Vec, + pub sandbox_restart_counts: Vec, + pub sandbox_exit_codes: Vec>, + pub sandbox_next_restart_at: Vec, pub sandbox_notes: Vec, pub sandbox_detail_notes: Vec, /// Formatted labels for each sandbox (e.g., "env=prod,team=platform" or empty string). @@ -1019,6 +1023,10 @@ impl App { sandbox_ages: Vec::new(), sandbox_created: Vec::new(), sandbox_images: Vec::new(), + sandbox_restart_policies: Vec::new(), + sandbox_restart_counts: Vec::new(), + sandbox_exit_codes: Vec::new(), + sandbox_next_restart_at: Vec::new(), sandbox_notes: Vec::new(), sandbox_detail_notes: Vec::new(), sandbox_labels: Vec::new(), @@ -3523,6 +3531,10 @@ impl App { self.sandbox_ages.clear(); self.sandbox_created.clear(); self.sandbox_images.clear(); + self.sandbox_restart_policies.clear(); + self.sandbox_restart_counts.clear(); + self.sandbox_exit_codes.clear(); + self.sandbox_next_restart_at.clear(); self.sandbox_notes.clear(); self.sandbox_detail_notes.clear(); self.sandbox_labels.clear(); diff --git a/crates/openshell-tui/src/lib.rs b/crates/openshell-tui/src/lib.rs index 0ede05d87b..f235c4e003 100644 --- a/crates/openshell-tui/src/lib.rs +++ b/crates/openshell-tui/src/lib.rs @@ -23,8 +23,8 @@ use miette::{IntoDiagnostic, Result}; use openshell_bootstrap::list_gateways_with_source; use openshell_core::auth::EdgeAuthInterceptor; use openshell_core::metadata::{ObjectId, ObjectLabels, ObjectName, ObjectWorkspace}; -use openshell_core::proto::SandboxPhase; use openshell_core::proto::open_shell_client::OpenShellClient; +use openshell_core::proto::{SandboxPhase, SandboxRestartPolicy}; use ratatui::Terminal; use ratatui::backend::CrosstermBackend; use tokio::sync::mpsc; @@ -2767,6 +2767,41 @@ fn apply_sandbox_refresh(app: &mut App, sandboxes: Vec "on-failure", + Some(SandboxRestartPolicy::Always) => "always", + _ => "never", + } + .to_string() + }) + .collect(); + app.sandbox_restart_counts = sandboxes + .iter() + .map(|s| s.status.as_ref().map_or(0, |status| status.restart_count)) + .collect(); + app.sandbox_exit_codes = sandboxes + .iter() + .map(|s| s.status.as_ref().and_then(|status| status.exit_code)) + .collect(); + app.sandbox_next_restart_at = sandboxes + .iter() + .map(|s| { + format_timestamp( + s.status + .as_ref() + .and_then(|status| status.next_restart_time.as_ref()) + .and_then(|time| openshell_core::time::timestamp_to_millis(time).ok()) + .unwrap_or_default(), + ) + }) + .collect(); app.sandbox_images = sandboxes .iter() .map(|s| { diff --git a/crates/openshell-tui/src/ui/sandbox_detail.rs b/crates/openshell-tui/src/ui/sandbox_detail.rs index 341e46c71a..552bda6a59 100644 --- a/crates/openshell-tui/src/ui/sandbox_detail.rs +++ b/crates/openshell-tui/src/ui/sandbox_detail.rs @@ -8,7 +8,7 @@ use ratatui::widgets::{Block, Borders, Padding, Paragraph}; use crate::app::App; -const BASE_CONTENT_ROWS: u16 = 6; +const BASE_CONTENT_ROWS: u16 = 7; const BORDER_ROWS: u16 = 2; fn pending_draft_count(app: &App) -> usize { @@ -113,6 +113,32 @@ pub fn draw(frame: &mut Frame<'_>, app: &App, area: Rect) { Span::styled(age, t.text), ]); + let restart_policy = app + .sandbox_restart_policies + .get(idx) + .map_or("never", String::as_str); + let restart_count = app.sandbox_restart_counts.get(idx).copied().unwrap_or(0); + let exit_code = app + .sandbox_exit_codes + .get(idx) + .copied() + .flatten() + .map_or_else(|| "-".to_string(), |code| code.to_string()); + let next_restart = app + .sandbox_next_restart_at + .get(idx) + .map_or("-", String::as_str); + let row_restart = Line::from(vec![ + Span::styled(" Restart: ", t.muted), + Span::styled(restart_policy, t.text), + Span::styled(" Count: ", t.muted), + Span::styled(restart_count.to_string(), t.text), + Span::styled(" Exit: ", t.muted), + Span::styled(exit_code, t.text), + Span::styled(" Next: ", t.muted), + Span::styled(next_restart, t.text), + ]); + // Row 3: Labels let labels_str = app .sandbox_labels @@ -146,7 +172,7 @@ pub fn draw(frame: &mut Frame<'_>, app: &App, area: Rect) { Span::styled(providers_str, t.text), ]); - let mut lines = vec![row1, row2, row3, row4, row5]; + let mut lines = vec![row1, row2, row_restart, row3, row4, row5]; lines.extend( note_lines(app, area.width) .into_iter() diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 847bbf3bf5..b20756c2e3 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -20,8 +20,8 @@ Create a sandbox with a single command. For example, to create a sandbox with Cl openshell sandbox create --from registry.example.com/your-org/claude-agent:latest -- claude ``` -The trailing command is the sandbox's canonical main process. OpenShell starts -it once, streams its output, and returns its exit status. Exit code 0 leaves a +The trailing command is the sandbox's canonical main process. OpenShell streams +its output and returns its exit status. By default, exit code 0 leaves a retained sandbox in `Completed`; a nonzero exit leaves it in `Error` with a `MainProcessFailed` condition. With no trailing command, OpenShell starts a login shell in a retained pseudo-terminal: `/bin/bash -l` when the image @@ -33,6 +33,20 @@ attaching: openshell sandbox create --name worker --detach -- ./worker ``` +Use `--restart-policy on-failure` to replace the runtime after a nonzero main +process exit, or `--restart-policy always` to replace it after every exit: + +```shell +openshell sandbox create --name worker --detach --restart-policy on-failure -- ./worker +``` + +The default policy is `never`. The gateway starts the first replacement as soon +as terminal output has been delivered. Repeated quick exits wait 1 second, then +double the delay up to 3 minutes. Running for 10 seconds resets the delay. A +replacement starts a new supervisor and main process. It retains the sandbox +identity and configuration; filesystem persistence follows the compute driver's stop and +start behavior. It does not restore process memory or terminal history. + Detached commands have no attachment grace period. When the command exits, OpenShell records its terminal phase immediately. For a foreground command, the create request declares one expected SSH attachment. OpenShell retains the @@ -853,7 +867,7 @@ remain in force. Timed-out records are retained even for ephemeral creates; use | Ready | The sandbox is running and its supervisor control session is connected. You can connect, execute commands, sync files, and view logs. | | Stopping | The gateway accepted a stop request and is stopping compute while retaining persistent state. | | Stopped | Compute was stopped explicitly and access is unavailable. | -| Starting | Compute is starting. The sandbox becomes usable only after a fresh supervisor session connects. | +| Starting | Compute is starting, or a selected main-process restart is waiting for backoff or a fresh supervisor session. Access resumes after the new session connects. | | Completed | The canonical main process exited with code 0. Its normalized result is available in `status.exit_code`. | | Error | The canonical main process failed, or sandbox infrastructure failed. Inspect the condition reason and `status.exit_code`. | | Deleting | The sandbox is being torn down. The system releases resources and purges credentials. | @@ -868,8 +882,12 @@ temporarily while its supervisor reconnects. Wait for the phase to return to The gateway records a successful canonical main-process exit as `Ready=False` with reason `MainProcessCompleted`. Nonzero and signal-normalized results use `MainProcessFailed` and the `Error` phase. It also sets `status.exit_code`; -signal exits use the standard `128 + signal` convention. Compute runtimes do -not automatically restart that process. +signal exits use the standard `128 + signal` convention. When restart policy +selects replacement, `status.restart_count` and `status.next_restart_time` +show the attempt number and next deadline. Inspect them with +`openshell sandbox get `. `sandbox stop` cancels a pending replacement; +an explicit later start resets the restart count. Compute runtimes do not +apply their own restart policy. ## Sandbox Runtimes diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index cd3d576b52..09a2771ff1 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -10,6 +10,11 @@ position: 4 Each gateway runs agent workloads on one compute runtime, selected by its compute driver. Pick the runtime that matches the isolation boundary and infrastructure you need. The CLI workflow stays the same across runtimes: you create, connect to, stop, start, and delete sandboxes through the gateway. +The gateway owns main-process restart policy. Docker, Podman, Kubernetes, and +MicroVM native restart mechanisms remain disabled. When policy selects a +replacement, the gateway stops and starts the compute resource while retaining +the sandbox record and driver-supported persistent storage. + | Runtime | Driver | Use when | |---|---|---| | [Docker](#docker-driver) | `docker` | Local development and single-machine gateways. | diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs index 4df9f24eba..60234e7b3e 100644 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ b/e2e/rust/tests/sandbox_lifecycle.rs @@ -18,6 +18,7 @@ use tokio::time::{Instant, sleep}; const SANDBOX_PRESENCE_TIMEOUT: Duration = Duration::from_secs(30); const SANDBOX_LIST_POLL_INTERVAL: Duration = Duration::from_millis(500); +const SANDBOX_RESTART_TIMEOUT: Duration = Duration::from_secs(60); fn normalize_output(output: &str) -> String { let stripped = strip_ansi(output).replace('\r', ""); @@ -1266,6 +1267,71 @@ async fn canonical_main_exit_255_is_not_retried_as_transport_failure() { #[tokio::test] #[serial(sandbox_lifecycle)] +async fn on_failure_policy_replaces_runtime_and_preserves_workspace() { + const FIRST_MARKER: &str = "initial-main-ready"; + const SCRIPT: &str = r#" +marker=/sandbox/.openshell-restart-e2e +if [ -e "$marker" ]; then + printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run + printf 'replacement-main-ready\n' + sleep 300 +else + touch "$marker" + printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run + printf 'initial-main-ready\n' + sleep 2 + exit 17 +fi +"#; + + let mut sandbox = SandboxGuard::create_keep_with_args( + &["--restart-policy", "on-failure"], + &["sh", "-lc", SCRIPT], + FIRST_MARKER, + ) + .await + .expect("create sandbox with OnFailure restart policy"); + + let deadline = Instant::now() + SANDBOX_RESTART_TIMEOUT; + let mut observed_replacement_starting = false; + let final_details = loop { + let details = sandbox_details(&sandbox.name).await; + observed_replacement_starting |= details.contains("Phase: Starting"); + if observed_replacement_starting + && details.contains("Phase: Ready") + && details.contains("Restart count: 1") + { + break details; + } + assert!( + Instant::now() < deadline, + "sandbox did not complete its policy-driven restart within \ + {SANDBOX_RESTART_TIMEOUT:?}; last details:\n{details}" + ); + sleep(Duration::from_millis(250)).await; + }; + + assert!( + final_details.contains("Restart policy: on-failure"), + "restart policy should remain visible after replacement:\n{final_details}" + ); + let runs = sandbox + .exec(&["cat", "/sandbox/initial-run", "/sandbox/replacement-run"]) + .await + .expect("replacement sandbox should retain the first run's workspace"); + assert!( + runs.contains("initial-"), + "missing initial run marker:\n{runs}" + ); + assert!( + runs.contains("replacement-"), + "missing replacement run marker:\n{runs}" + ); + + sandbox.cleanup().await; +} + +#[tokio::test] async fn sandbox_create_with_no_keep_cleans_up_after_tty_command() { let name = format!("tty-{:015x}", rand::random::() & 0x0fff_ffff_ffff_ffff); // Capture startup diagnostics before --no-keep removes a failed container. diff --git a/proto/openshell.proto b/proto/openshell.proto index a836e169bf..4323277886 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -1045,6 +1045,9 @@ message SandboxSpec { // Gateway-owned attachment identity, changed atomically with the provider set. // Equality only: detach and reattach must not revive an older receipt. string provider_attachment_epoch = 14; + // Gateway-owned policy for replacing the sandbox runtime after the canonical + // main process exits. Unspecified is normalized to Never before persistence. + SandboxRestartPolicy restart_policy = 15; } message ResourceRequirements { @@ -1170,9 +1173,8 @@ message SandboxStatus { // Supervisor instance currently associated with the canonical main process. // The gateway uses this to reject stale exit reports after a restart. string main_process_instance_id = 8; - // Normalized main process result. Signal exits use 128 + signal number. - // Presence indicates that the canonical main process exited. Exit code 0 - // produces Completed; nonzero and signal-normalized exits produce Error. + // Most recent normalized main process result. Signal exits use 128 + signal number. + // Cleared when a replacement main process becomes ready. optional int32 exit_code = 9; // Last accepted network result for each configured tool server endpoint. // Currently populated for MCP-over-HTTP endpoints. These passive results @@ -1184,6 +1186,12 @@ message SandboxStatus { optional bool configuration_activated = 12; // Gateway-owned repair window. Retained after timeout for inspection and retry. SandboxProvisioning provisioning = 13; + // Consecutive policy-driven restart number in the current crash loop. + uint32 restart_count = 14; + // Deadline for the next restart attempt or recovery check. + google.protobuf.Timestamp next_restart_time = 15; + // Time when the current main process became ready. + google.protobuf.Timestamp main_process_started_time = 16; } // User-facing sandbox condition derived from platform or gateway observations. @@ -1218,6 +1226,8 @@ enum SandboxPhase { SANDBOX_PHASE_STARTING = 8; // The canonical main process exited successfully and its result is final. SANDBOX_PHASE_COMPLETED = 9; + reserved 10; + reserved "SANDBOX_PHASE_RESTARTING"; } // Public platform event exposed on the sandbox watch stream. @@ -3552,6 +3562,16 @@ enum ProviderCredentialRefreshRecoveryAction { PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_INVESTIGATE = 4; } +// Policy applied when the canonical main process exits outside an intentional +// stop or delete operation. Kept after existing enums so generated enum +// descriptors remain stable. +enum SandboxRestartPolicy { + SANDBOX_RESTART_POLICY_UNSPECIFIED = 0; + SANDBOX_RESTART_POLICY_NEVER = 1; + SANDBOX_RESTART_POLICY_ON_FAILURE = 2; + SANDBOX_RESTART_POLICY_ALWAYS = 3; +} + // Workspace membership record. message WorkspaceMember { openshell.datamodel.v1.ObjectMeta metadata = 1; diff --git a/python/openshell/sandbox.py b/python/openshell/sandbox.py index d85d4c70fa..20d4734443 100644 --- a/python/openshell/sandbox.py +++ b/python/openshell/sandbox.py @@ -448,6 +448,9 @@ class SandboxStatusRef: phase: int current_policy_version: int exit_code: int | None = None + restart_count: int = 0 + next_restart_at_ms: int | None = None + main_process_started_at_ms: int | None = None @dataclass(frozen=True) @@ -1767,6 +1770,13 @@ def _sandbox_ref( exit_code=status.exit_code if status is not None and status.HasField("exit_code") else None, + restart_count=status.restart_count if status is not None else 0, + next_restart_at_ms=status.next_restart_time.ToMilliseconds() + if status is not None and status.HasField("next_restart_time") + else None, + main_process_started_at_ms=status.main_process_started_time.ToMilliseconds() + if status is not None and status.HasField("main_process_started_time") + else None, ), labels=sandbox.metadata.labels if sandbox.metadata else {}, created_from_workload_template=provenance, diff --git a/python/openshell/sandbox_test.py b/python/openshell/sandbox_test.py index 1d32ea3a80..7702b9fcd7 100644 --- a/python/openshell/sandbox_test.py +++ b/python/openshell/sandbox_test.py @@ -2719,10 +2719,16 @@ def test_sandbox_ref_retains_gateway_labels() -> None: def test_sandbox_ref_includes_main_process_result() -> None: proto = _make_sandbox_proto("sandbox-1", "job-1") proto.status.exit_code = 0 + proto.status.restart_count = 2 + proto.status.next_restart_time.FromMilliseconds(1_700_000_000_000) + proto.status.main_process_started_time.FromMilliseconds(1_699_999_000_000) status = _sandbox_ref(proto).status assert status.exit_code == 0 + assert status.restart_count == 2 + assert status.next_restart_at_ms == 1_700_000_000_000 + assert status.main_process_started_at_ms == 1_699_999_000_000 def test_returned_labels_are_immutable() -> None: diff --git a/sdk/go/openshell/v1/internal/converter/coverage_test.go b/sdk/go/openshell/v1/internal/converter/coverage_test.go index 2f90b73a5d..85342ce387 100644 --- a/sdk/go/openshell/v1/internal/converter/coverage_test.go +++ b/sdk/go/openshell/v1/internal/converter/coverage_test.go @@ -31,6 +31,7 @@ func TestConverterCoversAllProtoFields_SandboxSpec(t *testing.T) { "resource_requirements": true, "command": true, "tty": true, + "restart_policy": true, } // The gateway owns this identity. Provider status exposes it through the @@ -115,15 +116,18 @@ func TestConverterCoversAllProtoFields_SandboxStartup(t *testing.T) { func TestConverterCoversAllProtoFields_SandboxStatus(t *testing.T) { handled := fieldSet{ - "agent_pod": true, - "agent_fd": true, - "sandbox_fd": true, - "phase": true, - "conditions": true, - "endpoint_statuses": true, - "current_policy_version": true, - "exit_code": true, - "configuration_admission": true, + "agent_pod": true, + "agent_fd": true, + "sandbox_fd": true, + "phase": true, + "conditions": true, + "endpoint_statuses": true, + "current_policy_version": true, + "exit_code": true, + "configuration_admission": true, + "restart_count": true, + "next_restart_time": true, + "main_process_started_time": true, } // The instance ID coordinates internal gateway/supervisor lifecycle // fencing. The first-activation marker governs static policy repair. diff --git a/sdk/go/openshell/v1/internal/converter/sandbox.go b/sdk/go/openshell/v1/internal/converter/sandbox.go index 1f773c100e..7d81a7c828 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox.go @@ -56,10 +56,11 @@ func SandboxFromProto(s *pb.Sandbox) *types.Sandbox { func sandboxSpecFromProto(spec *pb.SandboxSpec) types.SandboxSpec { result := types.SandboxSpec{ - LogLevel: spec.GetLogLevel(), - Environment: CopyStringMap(spec.GetEnvironment()), - Providers: CopyStringSlice(spec.GetProviders()), - Policy: SandboxPolicyFromProto(spec.GetPolicy()), + LogLevel: spec.GetLogLevel(), + Environment: CopyStringMap(spec.GetEnvironment()), + Providers: CopyStringSlice(spec.GetProviders()), + Policy: SandboxPolicyFromProto(spec.GetPolicy()), + RestartPolicy: SandboxRestartPolicyFromProto(spec.GetRestartPolicy()), } if tmpl := spec.GetTemplate(); tmpl != nil { @@ -143,6 +144,9 @@ func sandboxStatusFromProto(status *pb.SandboxStatus) types.SandboxStatus { Error: admission.GetError(), } } + result.RestartCount = status.GetRestartCount() + result.NextRestartAtMs = MillisFromProto(status.GetNextRestartTime()) + result.MainProcessStartedAtMs = MillisFromProto(status.GetMainProcessStartedTime()) return result } @@ -292,10 +296,35 @@ func SandboxSpecToProto(spec *types.SandboxSpec) *pb.SandboxSpec { result.Command = CopyStringSlice(spec.Command) result.Tty = spec.TTY + result.RestartPolicy = SandboxRestartPolicyToProto(spec.RestartPolicy) return result } +// SandboxRestartPolicyFromProto converts the restart policy to the curated SDK type. +func SandboxRestartPolicyFromProto(policy pb.SandboxRestartPolicy) types.SandboxRestartPolicy { + switch policy { + case pb.SandboxRestartPolicy_SANDBOX_RESTART_POLICY_ON_FAILURE: + return types.SandboxRestartOnFailure + case pb.SandboxRestartPolicy_SANDBOX_RESTART_POLICY_ALWAYS: + return types.SandboxRestartAlways + default: + return types.SandboxRestartNever + } +} + +// SandboxRestartPolicyToProto converts the curated SDK restart policy to protobuf. +func SandboxRestartPolicyToProto(policy types.SandboxRestartPolicy) pb.SandboxRestartPolicy { + switch policy { + case types.SandboxRestartOnFailure: + return pb.SandboxRestartPolicy_SANDBOX_RESTART_POLICY_ON_FAILURE + case types.SandboxRestartAlways: + return pb.SandboxRestartPolicy_SANDBOX_RESTART_POLICY_ALWAYS + default: + return pb.SandboxRestartPolicy_SANDBOX_RESTART_POLICY_NEVER + } +} + // SandboxSpecToProtoChecked converts an SDK SandboxSpec and reports values // that protobuf Struct cannot represent instead of silently dropping them. func SandboxSpecToProtoChecked(spec *types.SandboxSpec) (*pb.SandboxSpec, error) { diff --git a/sdk/go/openshell/v1/internal/converter/sandbox_test.go b/sdk/go/openshell/v1/internal/converter/sandbox_test.go index 598029031a..2099b5b65b 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox_test.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox_test.go @@ -87,21 +87,25 @@ func TestSandboxFromProto(t *testing.T) { Count: &gpuCount, }, }, - Command: []string{"/opt/agent", "--serve"}, - Tty: false, + Command: []string{"/opt/agent", "--serve"}, + Tty: false, + RestartPolicy: pb.SandboxRestartPolicy_SANDBOX_RESTART_POLICY_ON_FAILURE, }, CreatedFromWorkloadTemplate: &pb.SandboxWorkloadTemplateProvenance{ Name: "gpu-kata", ResourceVersion: "7", }, Status: &pb.SandboxStatus{ - AgentPod: "agent-pod-xyz", - AgentFd: "fd-agent", - SandboxFd: "fd-sandbox", - Phase: pb.SandboxPhase_SANDBOX_PHASE_READY, - CurrentPolicyVersion: 7, - MainProcessInstanceId: "instance-1", - ExitCode: &exitCode, + AgentPod: "agent-pod-xyz", + AgentFd: "fd-agent", + SandboxFd: "fd-sandbox", + Phase: pb.SandboxPhase_SANDBOX_PHASE_READY, + CurrentPolicyVersion: 7, + MainProcessInstanceId: "instance-1", + ExitCode: &exitCode, + RestartCount: 3, + NextRestartTime: TimestampFromMillis(1700000070000), + MainProcessStartedTime: TimestampFromMillis(1700000010000), Conditions: []*pb.SandboxCondition{ { Type: "Ready", @@ -139,6 +143,7 @@ func TestSandboxFromProto(t *testing.T) { assert.Equal(t, uint32(2), *s.Spec.GPUCount) assert.Equal(t, []string{"/opt/agent", "--serve"}, s.Spec.Command) assert.False(t, s.Spec.TTY) + assert.Equal(t, v1.SandboxRestartOnFailure, s.Spec.RestartPolicy) // Template require.NotNil(t, s.Spec.Template) @@ -170,6 +175,9 @@ func TestSandboxFromProto(t *testing.T) { assert.Equal(t, "2024-01-01T00:00:00Z", s.Status.Conditions[0].LastTransitionTime) require.NotNil(t, s.Status.ExitCode) assert.Equal(t, int32(0), *s.Status.ExitCode) + assert.Equal(t, uint32(3), s.Status.RestartCount) + assert.Equal(t, int64(1700000070000), s.Status.NextRestartAtMs) + assert.Equal(t, int64(1700000010000), s.Status.MainProcessStartedAtMs) } func TestSandboxFromProto_TemplateResourcesDeepCopy(t *testing.T) { diff --git a/sdk/go/openshell/v1/types.go b/sdk/go/openshell/v1/types.go index 6226aa0707..e565d966cb 100644 --- a/sdk/go/openshell/v1/types.go +++ b/sdk/go/openshell/v1/types.go @@ -23,6 +23,16 @@ const ( SandboxCompleted = types.SandboxCompleted ) +// SandboxRestartPolicy controls replacement after the canonical main process exits. +type SandboxRestartPolicy = types.SandboxRestartPolicy + +// Sandbox restart policy values. +const ( + SandboxRestartNever = types.SandboxRestartNever + SandboxRestartOnFailure = types.SandboxRestartOnFailure + SandboxRestartAlways = types.SandboxRestartAlways +) + // EventType classifies watch events. type EventType = types.EventType diff --git a/sdk/go/openshell/v1/types/sandbox.go b/sdk/go/openshell/v1/types/sandbox.go index 1218bb6224..c8e7d43ef3 100644 --- a/sdk/go/openshell/v1/types/sandbox.go +++ b/sdk/go/openshell/v1/types/sandbox.go @@ -34,9 +34,10 @@ type SandboxSpec struct { GPU bool GPUCount *uint32 // Policy is the security policy for the sandbox. Nil means no policy specified. - Policy *SandboxPolicy - Command []string - TTY bool + Policy *SandboxPolicy + Command []string + TTY bool + RestartPolicy SandboxRestartPolicy } // SandboxTemplate defines the container template for a sandbox. @@ -123,6 +124,9 @@ type SandboxStatus struct { // last accepted network results, independently of sandbox readiness. EndpointStatuses []EndpointStatus ConfigurationAdmission *SandboxConfigurationAdmission + RestartCount uint32 + NextRestartAtMs int64 + MainProcessStartedAtMs int64 } // ConfigurationAdmissionState describes validation of an effective configuration. diff --git a/sdk/go/openshell/v1/types/types.go b/sdk/go/openshell/v1/types/types.go index 5b32ec3685..99d8e24bf9 100644 --- a/sdk/go/openshell/v1/types/types.go +++ b/sdk/go/openshell/v1/types/types.go @@ -21,6 +21,16 @@ const ( SandboxCompleted SandboxPhase = "Completed" ) +// SandboxRestartPolicy controls replacement after the canonical main process exits. +type SandboxRestartPolicy string + +// Sandbox restart policy values. +const ( + SandboxRestartNever SandboxRestartPolicy = "Never" + SandboxRestartOnFailure SandboxRestartPolicy = "OnFailure" + SandboxRestartAlways SandboxRestartPolicy = "Always" +) + // EventType classifies watch events. type EventType string diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 18d0b9bcde..6e1519314d 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -1005,6 +1005,61 @@ func (ProviderCredentialRefreshRecoveryAction) EnumDescriptor() ([]byte, []int) return file_openshell_proto_rawDescGZIP(), []int{15} } +// Policy applied when the canonical main process exits outside an intentional +// stop or delete operation. Kept after existing enums so generated enum +// descriptors remain stable. +type SandboxRestartPolicy int32 + +const ( + SandboxRestartPolicy_SANDBOX_RESTART_POLICY_UNSPECIFIED SandboxRestartPolicy = 0 + SandboxRestartPolicy_SANDBOX_RESTART_POLICY_NEVER SandboxRestartPolicy = 1 + SandboxRestartPolicy_SANDBOX_RESTART_POLICY_ON_FAILURE SandboxRestartPolicy = 2 + SandboxRestartPolicy_SANDBOX_RESTART_POLICY_ALWAYS SandboxRestartPolicy = 3 +) + +// Enum value maps for SandboxRestartPolicy. +var ( + SandboxRestartPolicy_name = map[int32]string{ + 0: "SANDBOX_RESTART_POLICY_UNSPECIFIED", + 1: "SANDBOX_RESTART_POLICY_NEVER", + 2: "SANDBOX_RESTART_POLICY_ON_FAILURE", + 3: "SANDBOX_RESTART_POLICY_ALWAYS", + } + SandboxRestartPolicy_value = map[string]int32{ + "SANDBOX_RESTART_POLICY_UNSPECIFIED": 0, + "SANDBOX_RESTART_POLICY_NEVER": 1, + "SANDBOX_RESTART_POLICY_ON_FAILURE": 2, + "SANDBOX_RESTART_POLICY_ALWAYS": 3, + } +) + +func (x SandboxRestartPolicy) Enum() *SandboxRestartPolicy { + p := new(SandboxRestartPolicy) + *p = x + return p +} + +func (x SandboxRestartPolicy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SandboxRestartPolicy) Descriptor() protoreflect.EnumDescriptor { + return file_openshell_proto_enumTypes[16].Descriptor() +} + +func (SandboxRestartPolicy) Type() protoreflect.EnumType { + return &file_openshell_proto_enumTypes[16] +} + +func (x SandboxRestartPolicy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SandboxRestartPolicy.Descriptor instead. +func (SandboxRestartPolicy) EnumDescriptor() ([]byte, []int) { + return file_openshell_proto_rawDescGZIP(), []int{16} +} + // Result of a public delete, membership removal, or session revocation. // Default requests return NOT_FOUND for a missing target. With allow_missing, // only a missing target becomes ALREADY_ABSENT; parent lookup, authorization, @@ -1052,11 +1107,11 @@ func (x DeletionOutcome) String() string { } func (DeletionOutcome) Descriptor() protoreflect.EnumDescriptor { - return file_openshell_proto_enumTypes[16].Descriptor() + return file_openshell_proto_enumTypes[17].Descriptor() } func (DeletionOutcome) Type() protoreflect.EnumType { - return &file_openshell_proto_enumTypes[16] + return &file_openshell_proto_enumTypes[17] } func (x DeletionOutcome) Number() protoreflect.EnumNumber { @@ -1065,7 +1120,7 @@ func (x DeletionOutcome) Number() protoreflect.EnumNumber { // Deprecated: Use DeletionOutcome.Descriptor instead. func (DeletionOutcome) EnumDescriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{16} + return file_openshell_proto_rawDescGZIP(), []int{17} } // Last observed network result for a configured external tool endpoint. @@ -1126,11 +1181,11 @@ func (x EndpointResult) String() string { } func (EndpointResult) Descriptor() protoreflect.EnumDescriptor { - return file_openshell_proto_enumTypes[17].Descriptor() + return file_openshell_proto_enumTypes[18].Descriptor() } func (EndpointResult) Type() protoreflect.EnumType { - return &file_openshell_proto_enumTypes[17] + return &file_openshell_proto_enumTypes[18] } func (x EndpointResult) Number() protoreflect.EnumNumber { @@ -1139,7 +1194,7 @@ func (x EndpointResult) Number() protoreflect.EnumNumber { // Deprecated: Use EndpointResult.Descriptor instead. func (EndpointResult) EnumDescriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{17} + return file_openshell_proto_rawDescGZIP(), []int{18} } // IssueSandboxToken request. Empty body; identity is established by the @@ -2250,8 +2305,11 @@ type SandboxSpec struct { // Gateway-owned attachment identity, changed atomically with the provider set. // Equality only: detach and reattach must not revive an older receipt. ProviderAttachmentEpoch string `protobuf:"bytes,14,opt,name=provider_attachment_epoch,json=providerAttachmentEpoch,proto3" json:"provider_attachment_epoch,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Gateway-owned policy for replacing the sandbox runtime after the canonical + // main process exits. Unspecified is normalized to Never before persistence. + RestartPolicy SandboxRestartPolicy `protobuf:"varint,15,opt,name=restart_policy,json=restartPolicy,proto3,enum=openshell.v1.SandboxRestartPolicy" json:"restart_policy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxSpec) Reset() { @@ -2347,6 +2405,13 @@ func (x *SandboxSpec) GetProviderAttachmentEpoch() string { return "" } +func (x *SandboxSpec) GetRestartPolicy() SandboxRestartPolicy { + if x != nil { + return x.RestartPolicy + } + return SandboxRestartPolicy_SANDBOX_RESTART_POLICY_UNSPECIFIED +} + type ResourceRequirements struct { state protoimpl.MessageState `protogen:"open.v1"` // GPU requirements for the sandbox. Presence indicates a GPU request. @@ -2988,9 +3053,8 @@ type SandboxStatus struct { // Supervisor instance currently associated with the canonical main process. // The gateway uses this to reject stale exit reports after a restart. MainProcessInstanceId string `protobuf:"bytes,8,opt,name=main_process_instance_id,json=mainProcessInstanceId,proto3" json:"main_process_instance_id,omitempty"` - // Normalized main process result. Signal exits use 128 + signal number. - // Presence indicates that the canonical main process exited. Exit code 0 - // produces Completed; nonzero and signal-normalized exits produce Error. + // Most recent normalized main process result. Signal exits use 128 + signal number. + // Cleared when a replacement main process becomes ready. ExitCode *int32 `protobuf:"varint,9,opt,name=exit_code,json=exitCode,proto3,oneof" json:"exit_code,omitempty"` // Last accepted network result for each configured tool server endpoint. // Currently populated for MCP-over-HTTP endpoints. These passive results @@ -3001,9 +3065,15 @@ type SandboxStatus struct { // Durable first-acceptance marker. Absent on legacy records; never reset by restart. ConfigurationActivated *bool `protobuf:"varint,12,opt,name=configuration_activated,json=configurationActivated,proto3,oneof" json:"configuration_activated,omitempty"` // Gateway-owned repair window. Retained after timeout for inspection and retry. - Provisioning *SandboxProvisioning `protobuf:"bytes,13,opt,name=provisioning,proto3" json:"provisioning,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Provisioning *SandboxProvisioning `protobuf:"bytes,13,opt,name=provisioning,proto3" json:"provisioning,omitempty"` + // Consecutive policy-driven restart number in the current crash loop. + RestartCount uint32 `protobuf:"varint,14,opt,name=restart_count,json=restartCount,proto3" json:"restart_count,omitempty"` + // Deadline for the next restart attempt or recovery check. + NextRestartTime *timestamppb.Timestamp `protobuf:"bytes,15,opt,name=next_restart_time,json=nextRestartTime,proto3" json:"next_restart_time,omitempty"` + // Time when the current main process became ready. + MainProcessStartedTime *timestamppb.Timestamp `protobuf:"bytes,16,opt,name=main_process_started_time,json=mainProcessStartedTime,proto3" json:"main_process_started_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxStatus) Reset() { @@ -3120,6 +3190,27 @@ func (x *SandboxStatus) GetProvisioning() *SandboxProvisioning { return nil } +func (x *SandboxStatus) GetRestartCount() uint32 { + if x != nil { + return x.RestartCount + } + return 0 +} + +func (x *SandboxStatus) GetNextRestartTime() *timestamppb.Timestamp { + if x != nil { + return x.NextRestartTime + } + return nil +} + +func (x *SandboxStatus) GetMainProcessStartedTime() *timestamppb.Timestamp { + if x != nil { + return x.MainProcessStartedTime + } + return nil +} + // User-facing sandbox condition derived from platform or gateway observations. type SandboxCondition struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -17640,7 +17731,7 @@ const file_openshell_proto_rawDesc = "" + "\bmetadata\x18\x01 \x01(\v2\".openshell.datamodel.v1.ObjectMetaR\bmetadata\x12-\n" + "\x04spec\x18\x02 \x01(\v2\x19.openshell.v1.SandboxSpecR\x04spec\x123\n" + "\x06status\x18\x03 \x01(\v2\x1b.openshell.v1.SandboxStatusR\x06status\x12t\n" + - "\x1ecreated_from_workload_template\x18\x14 \x01(\v2/.openshell.v1.SandboxWorkloadTemplateProvenanceR\x1bcreatedFromWorkloadTemplateJ\x04\b\x04\x10\x05J\x04\b\x05\x10\x06R\x05phaseR\x16current_policy_version\"\xbf\x04\n" + + "\x1ecreated_from_workload_template\x18\x14 \x01(\v2/.openshell.v1.SandboxWorkloadTemplateProvenanceR\x1bcreatedFromWorkloadTemplateJ\x04\b\x04\x10\x05J\x04\b\x05\x10\x06R\x05phaseR\x16current_policy_version\"\x8a\x05\n" + "\vSandboxSpec\x12\x1b\n" + "\tlog_level\x18\x01 \x01(\tR\blogLevel\x12L\n" + "\venvironment\x18\x05 \x03(\v2*.openshell.v1.SandboxSpec.EnvironmentEntryR\venvironment\x129\n" + @@ -17650,7 +17741,8 @@ const file_openshell_proto_rawDesc = "" + "\x15resource_requirements\x18\t \x01(\v2\".openshell.v1.ResourceRequirementsR\x14resourceRequirements\x12\x18\n" + "\acommand\x18\f \x03(\tR\acommand\x12\x10\n" + "\x03tty\x18\r \x01(\bR\x03tty\x12:\n" + - "\x19provider_attachment_epoch\x18\x0e \x01(\tR\x17providerAttachmentEpoch\x1a>\n" + + "\x19provider_attachment_epoch\x18\x0e \x01(\tR\x17providerAttachmentEpoch\x12I\n" + + "\x0erestart_policy\x18\x0f \x01(\x0e2\".openshell.v1.SandboxRestartPolicyR\rrestartPolicy\x1a>\n" + "\x10EnvironmentEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01J\x04\b\n" + @@ -17708,7 +17800,7 @@ const file_openshell_proto_rawDesc = "" + "\tmax_burst\x18\x02 \x01(\rR\bmaxBurst\"b\n" + "!SandboxWorkloadTemplateProvenance\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12)\n" + - "\x10resource_version\x18\x02 \x01(\tR\x0fresourceVersion\"\xc9\x05\n" + + "\x10resource_version\x18\x02 \x01(\tR\x0fresourceVersion\"\x8d\a\n" + "\rSandboxStatus\x12\x1b\n" + "\tagent_pod\x18\x02 \x01(\tR\bagentPod\x12\x19\n" + "\bagent_fd\x18\x03 \x01(\tR\aagentFd\x12\x1d\n" + @@ -17725,7 +17817,10 @@ const file_openshell_proto_rawDesc = "" + " \x03(\v2\x1c.openshell.v1.EndpointStatusR\x10endpointStatuses\x12d\n" + "\x17configuration_admission\x18\v \x01(\v2+.openshell.v1.SandboxConfigurationAdmissionR\x16configurationAdmission\x12<\n" + "\x17configuration_activated\x18\f \x01(\bH\x01R\x16configurationActivated\x88\x01\x01\x12E\n" + - "\fprovisioning\x18\r \x01(\v2!.openshell.v1.SandboxProvisioningR\fprovisioningB\f\n" + + "\fprovisioning\x18\r \x01(\v2!.openshell.v1.SandboxProvisioningR\fprovisioning\x12#\n" + + "\rrestart_count\x18\x0e \x01(\rR\frestartCount\x12F\n" + + "\x11next_restart_time\x18\x0f \x01(\v2\x1a.google.protobuf.TimestampR\x0fnextRestartTime\x12U\n" + + "\x19main_process_started_time\x18\x10 \x01(\v2\x1a.google.protobuf.TimestampR\x16mainProcessStartedTimeB\f\n" + "\n" + "_exit_codeB\x1a\n" + "\x18_configuration_activated\"\xd1\x01\n" + @@ -18894,7 +18989,7 @@ const file_openshell_proto_rawDesc = "" + "\x1dEXTENSION_KIND_COMPUTE_DRIVER\x10\x01\x12$\n" + " EXTENSION_KIND_CREDENTIAL_DRIVER\x10\x02\x12&\n" + "\"EXTENSION_KIND_GATEWAY_INTERCEPTOR\x10\x03\x12(\n" + - "$EXTENSION_KIND_SUPERVISOR_MIDDLEWARE\x10\x04*\xa6\x02\n" + + "$EXTENSION_KIND_SUPERVISOR_MIDDLEWARE\x10\x04*\xc6\x02\n" + "\fSandboxPhase\x12\x1d\n" + "\x19SANDBOX_PHASE_UNSPECIFIED\x10\x00\x12\x1e\n" + "\x1aSANDBOX_PHASE_PROVISIONING\x10\x01\x12\x17\n" + @@ -18905,7 +19000,9 @@ const file_openshell_proto_rawDesc = "" + "\x16SANDBOX_PHASE_STOPPING\x10\x06\x12\x19\n" + "\x15SANDBOX_PHASE_STOPPED\x10\a\x12\x1a\n" + "\x16SANDBOX_PHASE_STARTING\x10\b\x12\x1b\n" + - "\x17SANDBOX_PHASE_COMPLETED\x10\t*\xcb\x01\n" + + "\x17SANDBOX_PHASE_COMPLETED\x10\t\"\x04\b\n" + + "\x10\n" + + "*\x18SANDBOX_PHASE_RESTARTING*\xcb\x01\n" + "\x14ProviderMutationKind\x12&\n" + "\"PROVIDER_MUTATION_KIND_UNSPECIFIED\x10\x00\x12!\n" + "\x1dPROVIDER_MUTATION_KIND_ATTACH\x10\x01\x12!\n" + @@ -19007,7 +19104,12 @@ const file_openshell_proto_rawDesc = "" + "1PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_RETRY\x10\x01\x12;\n" + "7PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_REAUTHORIZE\x10\x02\x12A\n" + "=PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_FIX_CONFIGURATION\x10\x03\x12;\n" + - "7PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_INVESTIGATE\x10\x04*\x97\x01\n" + + "7PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_INVESTIGATE\x10\x04*\xaa\x01\n" + + "\x14SandboxRestartPolicy\x12&\n" + + "\"SANDBOX_RESTART_POLICY_UNSPECIFIED\x10\x00\x12 \n" + + "\x1cSANDBOX_RESTART_POLICY_NEVER\x10\x01\x12%\n" + + "!SANDBOX_RESTART_POLICY_ON_FAILURE\x10\x02\x12!\n" + + "\x1dSANDBOX_RESTART_POLICY_ALWAYS\x10\x03*\x97\x01\n" + "\x0fDeletionOutcome\x12 \n" + "\x1cDELETION_OUTCOME_UNSPECIFIED\x10\x00\x12\x1e\n" + "\x1aDELETION_OUTCOME_COMPLETED\x10\x01\x12\x1d\n" + @@ -19207,7 +19309,7 @@ func file_openshell_proto_rawDescGZIP() []byte { return file_openshell_proto_rawDescData } -var file_openshell_proto_enumTypes = make([]protoimpl.EnumInfo, 18) +var file_openshell_proto_enumTypes = make([]protoimpl.EnumInfo, 19) var file_openshell_proto_msgTypes = make([]protoimpl.MessageInfo, 253) var file_openshell_proto_goTypes = []any{ (ExtensionKind)(0), // 0: openshell.v1.ExtensionKind @@ -19226,767 +19328,771 @@ var file_openshell_proto_goTypes = []any{ (ServiceStatus)(0), // 13: openshell.v1.ServiceStatus (WorkspaceRole)(0), // 14: openshell.v1.WorkspaceRole (ProviderCredentialRefreshRecoveryAction)(0), // 15: openshell.v1.ProviderCredentialRefreshRecoveryAction - (DeletionOutcome)(0), // 16: openshell.v1.DeletionOutcome - (EndpointResult)(0), // 17: openshell.v1.EndpointResult - (*IssueSandboxTokenRequest)(nil), // 18: openshell.v1.IssueSandboxTokenRequest - (*IssueSandboxTokenResponse)(nil), // 19: openshell.v1.IssueSandboxTokenResponse - (*RefreshSandboxTokenRequest)(nil), // 20: openshell.v1.RefreshSandboxTokenRequest - (*RefreshSandboxTokenResponse)(nil), // 21: openshell.v1.RefreshSandboxTokenResponse - (*HealthRequest)(nil), // 22: openshell.v1.HealthRequest - (*HealthResponse)(nil), // 23: openshell.v1.HealthResponse - (*GetCurrentUserRequest)(nil), // 24: openshell.v1.GetCurrentUserRequest - (*GetCurrentUserResponse)(nil), // 25: openshell.v1.GetCurrentUserResponse - (*GetGatewayInfoRequest)(nil), // 26: openshell.v1.GetGatewayInfoRequest - (*GetGatewayInfoResponse)(nil), // 27: openshell.v1.GetGatewayInfoResponse - (*NegotiatedExtensionInfo)(nil), // 28: openshell.v1.NegotiatedExtensionInfo - (*ComputeDriverInfo)(nil), // 29: openshell.v1.ComputeDriverInfo - (*ComputeDriverCapabilities)(nil), // 30: openshell.v1.ComputeDriverCapabilities - (*ResourceCapabilities)(nil), // 31: openshell.v1.ResourceCapabilities - (*CpuResourceCapabilities)(nil), // 32: openshell.v1.CpuResourceCapabilities - (*MemoryResourceCapabilities)(nil), // 33: openshell.v1.MemoryResourceCapabilities - (*GpuResourceCapabilities)(nil), // 34: openshell.v1.GpuResourceCapabilities - (*Sandbox)(nil), // 35: openshell.v1.Sandbox - (*SandboxSpec)(nil), // 36: openshell.v1.SandboxSpec - (*ResourceRequirements)(nil), // 37: openshell.v1.ResourceRequirements - (*GpuResourceRequirements)(nil), // 38: openshell.v1.GpuResourceRequirements - (*SandboxTemplate)(nil), // 39: openshell.v1.SandboxTemplate - (*SandboxWorkloadTemplate)(nil), // 40: openshell.v1.SandboxWorkloadTemplate - (*SandboxWorkloadTemplateSpec)(nil), // 41: openshell.v1.SandboxWorkloadTemplateSpec - (*SandboxWorkloadConfig)(nil), // 42: openshell.v1.SandboxWorkloadConfig - (*SandboxResources)(nil), // 43: openshell.v1.SandboxResources - (*SandboxServiceLevel)(nil), // 44: openshell.v1.SandboxServiceLevel - (*SandboxStartup)(nil), // 45: openshell.v1.SandboxStartup - (*SandboxWorkloadTemplateProvenance)(nil), // 46: openshell.v1.SandboxWorkloadTemplateProvenance - (*SandboxStatus)(nil), // 47: openshell.v1.SandboxStatus - (*SandboxCondition)(nil), // 48: openshell.v1.SandboxCondition - (*PlatformEvent)(nil), // 49: openshell.v1.PlatformEvent - (*CreateSandboxRequest)(nil), // 50: openshell.v1.CreateSandboxRequest - (*CreateSandboxTemplateRequest)(nil), // 51: openshell.v1.CreateSandboxTemplateRequest - (*GetSandboxTemplateRequest)(nil), // 52: openshell.v1.GetSandboxTemplateRequest - (*ListSandboxTemplatesRequest)(nil), // 53: openshell.v1.ListSandboxTemplatesRequest - (*DeleteSandboxTemplateRequest)(nil), // 54: openshell.v1.DeleteSandboxTemplateRequest - (*SandboxTemplateResponse)(nil), // 55: openshell.v1.SandboxTemplateResponse - (*ListSandboxTemplatesResponse)(nil), // 56: openshell.v1.ListSandboxTemplatesResponse - (*DeleteSandboxTemplateResponse)(nil), // 57: openshell.v1.DeleteSandboxTemplateResponse - (*BeginRootfsTarStagingRequest)(nil), // 58: openshell.v1.BeginRootfsTarStagingRequest - (*BeginRootfsTarStagingResponse)(nil), // 59: openshell.v1.BeginRootfsTarStagingResponse - (*GetSandboxRequest)(nil), // 60: openshell.v1.GetSandboxRequest - (*ListSandboxesRequest)(nil), // 61: openshell.v1.ListSandboxesRequest - (*ListSandboxProvidersRequest)(nil), // 62: openshell.v1.ListSandboxProvidersRequest - (*AttachSandboxProviderRequest)(nil), // 63: openshell.v1.AttachSandboxProviderRequest - (*DetachSandboxProviderRequest)(nil), // 64: openshell.v1.DetachSandboxProviderRequest - (*DeleteSandboxRequest)(nil), // 65: openshell.v1.DeleteSandboxRequest - (*StopSandboxRequest)(nil), // 66: openshell.v1.StopSandboxRequest - (*StartSandboxRequest)(nil), // 67: openshell.v1.StartSandboxRequest - (*SandboxResponse)(nil), // 68: openshell.v1.SandboxResponse - (*ListSandboxesResponse)(nil), // 69: openshell.v1.ListSandboxesResponse - (*ListSandboxProvidersResponse)(nil), // 70: openshell.v1.ListSandboxProvidersResponse - (*AttachSandboxProviderResponse)(nil), // 71: openshell.v1.AttachSandboxProviderResponse - (*DetachSandboxProviderResponse)(nil), // 72: openshell.v1.DetachSandboxProviderResponse - (*ProviderDesiredIdentity)(nil), // 73: openshell.v1.ProviderDesiredIdentity - (*ConfigSnapshotRevision)(nil), // 74: openshell.v1.ConfigSnapshotRevision - (*SandboxConfigRevision)(nil), // 75: openshell.v1.SandboxConfigRevision - (*ConfigUpdateOperation)(nil), // 76: openshell.v1.ConfigUpdateOperation - (*ProviderMutationReceipt)(nil), // 77: openshell.v1.ProviderMutationReceipt - (*ProviderReadinessObservation)(nil), // 78: openshell.v1.ProviderReadinessObservation - (*ProviderReadinessStatus)(nil), // 79: openshell.v1.ProviderReadinessStatus - (*GetSandboxProviderStatusRequest)(nil), // 80: openshell.v1.GetSandboxProviderStatusRequest - (*GetSandboxProviderStatusResponse)(nil), // 81: openshell.v1.GetSandboxProviderStatusResponse - (*ReportProviderReadinessRequest)(nil), // 82: openshell.v1.ReportProviderReadinessRequest - (*ReportProviderReadinessResponse)(nil), // 83: openshell.v1.ReportProviderReadinessResponse - (*DeleteSandboxResponse)(nil), // 84: openshell.v1.DeleteSandboxResponse - (*CreateSshSessionRequest)(nil), // 85: openshell.v1.CreateSshSessionRequest - (*CreateSshSessionResponse)(nil), // 86: openshell.v1.CreateSshSessionResponse - (*ExposeServiceRequest)(nil), // 87: openshell.v1.ExposeServiceRequest - (*GetServiceRequest)(nil), // 88: openshell.v1.GetServiceRequest - (*ListServicesRequest)(nil), // 89: openshell.v1.ListServicesRequest - (*ListServicesResponse)(nil), // 90: openshell.v1.ListServicesResponse - (*DeleteServiceRequest)(nil), // 91: openshell.v1.DeleteServiceRequest - (*DeleteServiceResponse)(nil), // 92: openshell.v1.DeleteServiceResponse - (*ServiceEndpoint)(nil), // 93: openshell.v1.ServiceEndpoint - (*ServiceEndpointResponse)(nil), // 94: openshell.v1.ServiceEndpointResponse - (*RevokeSshSessionRequest)(nil), // 95: openshell.v1.RevokeSshSessionRequest - (*RevokeSshSessionResponse)(nil), // 96: openshell.v1.RevokeSshSessionResponse - (*ExecSandboxRequest)(nil), // 97: openshell.v1.ExecSandboxRequest - (*ExecSandboxStdout)(nil), // 98: openshell.v1.ExecSandboxStdout - (*ExecSandboxStderr)(nil), // 99: openshell.v1.ExecSandboxStderr - (*ExecSandboxExit)(nil), // 100: openshell.v1.ExecSandboxExit - (*ExecSandboxEvent)(nil), // 101: openshell.v1.ExecSandboxEvent - (*TcpForwardInit)(nil), // 102: openshell.v1.TcpForwardInit - (*TcpForwardFrame)(nil), // 103: openshell.v1.TcpForwardFrame - (*ExecSandboxInput)(nil), // 104: openshell.v1.ExecSandboxInput - (*ExecSandboxWindowResize)(nil), // 105: openshell.v1.ExecSandboxWindowResize - (*SshSession)(nil), // 106: openshell.v1.SshSession - (*WatchSandboxRequest)(nil), // 107: openshell.v1.WatchSandboxRequest - (*SandboxStreamEvent)(nil), // 108: openshell.v1.SandboxStreamEvent - (*SandboxLogLine)(nil), // 109: openshell.v1.SandboxLogLine - (*SandboxStreamWarning)(nil), // 110: openshell.v1.SandboxStreamWarning - (*CreateProviderRequest)(nil), // 111: openshell.v1.CreateProviderRequest - (*GetProviderRequest)(nil), // 112: openshell.v1.GetProviderRequest - (*ListProvidersRequest)(nil), // 113: openshell.v1.ListProvidersRequest - (*UpdateProviderRequest)(nil), // 114: openshell.v1.UpdateProviderRequest - (*DeleteProviderRequest)(nil), // 115: openshell.v1.DeleteProviderRequest - (*ProviderResponse)(nil), // 116: openshell.v1.ProviderResponse - (*ListProvidersResponse)(nil), // 117: openshell.v1.ListProvidersResponse - (*ListProviderProfilesRequest)(nil), // 118: openshell.v1.ListProviderProfilesRequest - (*GetProviderProfileRequest)(nil), // 119: openshell.v1.GetProviderProfileRequest - (*ProviderProfileImportItem)(nil), // 120: openshell.v1.ProviderProfileImportItem - (*ProviderProfileDiagnostic)(nil), // 121: openshell.v1.ProviderProfileDiagnostic - (*ProviderCredentialTokenGrantAudienceOverride)(nil), // 122: openshell.v1.ProviderCredentialTokenGrantAudienceOverride - (*ProviderCredentialTokenGrantSubjectToken)(nil), // 123: openshell.v1.ProviderCredentialTokenGrantSubjectToken - (*ProviderCredentialTokenGrant)(nil), // 124: openshell.v1.ProviderCredentialTokenGrant - (*ProviderProfileCredential)(nil), // 125: openshell.v1.ProviderProfileCredential - (*ProviderCredentialRefreshMaterial)(nil), // 126: openshell.v1.ProviderCredentialRefreshMaterial - (*ProviderCredentialRefreshOutput)(nil), // 127: openshell.v1.ProviderCredentialRefreshOutput - (*ProviderCredentialRefresh)(nil), // 128: openshell.v1.ProviderCredentialRefresh - (*ProviderCredentialRefreshStatus)(nil), // 129: openshell.v1.ProviderCredentialRefreshStatus - (*ProviderProfileDiscovery)(nil), // 130: openshell.v1.ProviderProfileDiscovery - (*GetProviderRefreshStatusRequest)(nil), // 131: openshell.v1.GetProviderRefreshStatusRequest - (*GetProviderRefreshStatusResponse)(nil), // 132: openshell.v1.GetProviderRefreshStatusResponse - (*ConfigureProviderRefreshRequest)(nil), // 133: openshell.v1.ConfigureProviderRefreshRequest - (*ConfigureProviderRefreshResponse)(nil), // 134: openshell.v1.ConfigureProviderRefreshResponse - (*RotateProviderCredentialRequest)(nil), // 135: openshell.v1.RotateProviderCredentialRequest - (*RotateProviderCredentialResponse)(nil), // 136: openshell.v1.RotateProviderCredentialResponse - (*DeleteProviderRefreshRequest)(nil), // 137: openshell.v1.DeleteProviderRefreshRequest - (*DeleteProviderRefreshResponse)(nil), // 138: openshell.v1.DeleteProviderRefreshResponse - (*ProviderProfile)(nil), // 139: openshell.v1.ProviderProfile - (*ProviderProfileResponse)(nil), // 140: openshell.v1.ProviderProfileResponse - (*ListProviderProfilesResponse)(nil), // 141: openshell.v1.ListProviderProfilesResponse - (*ImportProviderProfilesRequest)(nil), // 142: openshell.v1.ImportProviderProfilesRequest - (*ImportProviderProfilesResponse)(nil), // 143: openshell.v1.ImportProviderProfilesResponse - (*UpdateProviderProfilesRequest)(nil), // 144: openshell.v1.UpdateProviderProfilesRequest - (*UpdateProviderProfilesResponse)(nil), // 145: openshell.v1.UpdateProviderProfilesResponse - (*LintProviderProfilesRequest)(nil), // 146: openshell.v1.LintProviderProfilesRequest - (*LintProviderProfilesResponse)(nil), // 147: openshell.v1.LintProviderProfilesResponse - (*DeleteProviderResponse)(nil), // 148: openshell.v1.DeleteProviderResponse - (*DeleteProviderProfileRequest)(nil), // 149: openshell.v1.DeleteProviderProfileRequest - (*DeleteProviderProfileResponse)(nil), // 150: openshell.v1.DeleteProviderProfileResponse - (*GetSandboxProviderEnvironmentRequest)(nil), // 151: openshell.v1.GetSandboxProviderEnvironmentRequest - (*StaticCredentialEndpointBinding)(nil), // 152: openshell.v1.StaticCredentialEndpointBinding - (*StaticCredentialBinding)(nil), // 153: openshell.v1.StaticCredentialBinding - (*GetSandboxProviderEnvironmentResponse)(nil), // 154: openshell.v1.GetSandboxProviderEnvironmentResponse - (*ExchangeProviderSubjectTokenRequest)(nil), // 155: openshell.v1.ExchangeProviderSubjectTokenRequest - (*ExchangeProviderSubjectTokenResponse)(nil), // 156: openshell.v1.ExchangeProviderSubjectTokenResponse - (*UpdateConfigRequest)(nil), // 157: openshell.v1.UpdateConfigRequest - (*PolicyMergeOperation)(nil), // 158: openshell.v1.PolicyMergeOperation - (*AddNetworkRule)(nil), // 159: openshell.v1.AddNetworkRule - (*RemoveNetworkEndpoint)(nil), // 160: openshell.v1.RemoveNetworkEndpoint - (*RemoveNetworkRule)(nil), // 161: openshell.v1.RemoveNetworkRule - (*L7RuleTarget)(nil), // 162: openshell.v1.L7RuleTarget - (*AddDenyRules)(nil), // 163: openshell.v1.AddDenyRules - (*AddAllowRules)(nil), // 164: openshell.v1.AddAllowRules - (*RemoveNetworkBinary)(nil), // 165: openshell.v1.RemoveNetworkBinary - (*UpdateConfigResponse)(nil), // 166: openshell.v1.UpdateConfigResponse - (*GetSandboxPolicyStatusRequest)(nil), // 167: openshell.v1.GetSandboxPolicyStatusRequest - (*GetSandboxPolicyStatusResponse)(nil), // 168: openshell.v1.GetSandboxPolicyStatusResponse - (*ListSandboxPoliciesRequest)(nil), // 169: openshell.v1.ListSandboxPoliciesRequest - (*ListSandboxPoliciesResponse)(nil), // 170: openshell.v1.ListSandboxPoliciesResponse - (*ReportPolicyStatusRequest)(nil), // 171: openshell.v1.ReportPolicyStatusRequest - (*ReportPolicyStatusResponse)(nil), // 172: openshell.v1.ReportPolicyStatusResponse - (*SandboxConfigurationAdmission)(nil), // 173: openshell.v1.SandboxConfigurationAdmission - (*ReportSandboxConfigurationRequest)(nil), // 174: openshell.v1.ReportSandboxConfigurationRequest - (*ReportSandboxConfigurationResponse)(nil), // 175: openshell.v1.ReportSandboxConfigurationResponse - (*SandboxPolicyRevision)(nil), // 176: openshell.v1.SandboxPolicyRevision - (*GetSandboxLogsRequest)(nil), // 177: openshell.v1.GetSandboxLogsRequest - (*PushSandboxLogsRequest)(nil), // 178: openshell.v1.PushSandboxLogsRequest - (*PushSandboxLogsResponse)(nil), // 179: openshell.v1.PushSandboxLogsResponse - (*GetSandboxLogsResponse)(nil), // 180: openshell.v1.GetSandboxLogsResponse - (*SupervisorMessage)(nil), // 181: openshell.v1.SupervisorMessage - (*GatewayMessage)(nil), // 182: openshell.v1.GatewayMessage - (*SupervisorHello)(nil), // 183: openshell.v1.SupervisorHello - (*SessionAccepted)(nil), // 184: openshell.v1.SessionAccepted - (*SessionRejected)(nil), // 185: openshell.v1.SessionRejected - (*SupervisorHeartbeat)(nil), // 186: openshell.v1.SupervisorHeartbeat - (*GatewayHeartbeat)(nil), // 187: openshell.v1.GatewayHeartbeat - (*ReportMainProcessExitRequest)(nil), // 188: openshell.v1.ReportMainProcessExitRequest - (*ReportMainProcessExitResponse)(nil), // 189: openshell.v1.ReportMainProcessExitResponse - (*FinalizeMainProcessExitRequest)(nil), // 190: openshell.v1.FinalizeMainProcessExitRequest - (*FinalizeMainProcessExitResponse)(nil), // 191: openshell.v1.FinalizeMainProcessExitResponse - (*RelayOpen)(nil), // 192: openshell.v1.RelayOpen - (*SshRelayTarget)(nil), // 193: openshell.v1.SshRelayTarget - (*TcpRelayTarget)(nil), // 194: openshell.v1.TcpRelayTarget - (*RelayInit)(nil), // 195: openshell.v1.RelayInit - (*RelayFrame)(nil), // 196: openshell.v1.RelayFrame - (*PeerRelayInit)(nil), // 197: openshell.v1.PeerRelayInit - (*PeerRelayFrame)(nil), // 198: openshell.v1.PeerRelayFrame - (*RelayOpenResult)(nil), // 199: openshell.v1.RelayOpenResult - (*RelayClose)(nil), // 200: openshell.v1.RelayClose - (*L7RequestSample)(nil), // 201: openshell.v1.L7RequestSample - (*DenialSummary)(nil), // 202: openshell.v1.DenialSummary - (*DenialGroupCount)(nil), // 203: openshell.v1.DenialGroupCount - (*NetworkActivitySummary)(nil), // 204: openshell.v1.NetworkActivitySummary - (*PolicyChunk)(nil), // 205: openshell.v1.PolicyChunk - (*DraftPolicyUpdate)(nil), // 206: openshell.v1.DraftPolicyUpdate - (*SubmitPolicyAnalysisRequest)(nil), // 207: openshell.v1.SubmitPolicyAnalysisRequest - (*SubmitPolicyAnalysisResponse)(nil), // 208: openshell.v1.SubmitPolicyAnalysisResponse - (*GetDraftPolicyRequest)(nil), // 209: openshell.v1.GetDraftPolicyRequest - (*GetDraftPolicyResponse)(nil), // 210: openshell.v1.GetDraftPolicyResponse - (*ApproveDraftChunkRequest)(nil), // 211: openshell.v1.ApproveDraftChunkRequest - (*ApproveDraftChunkResponse)(nil), // 212: openshell.v1.ApproveDraftChunkResponse - (*RejectDraftChunkRequest)(nil), // 213: openshell.v1.RejectDraftChunkRequest - (*RejectDraftChunkResponse)(nil), // 214: openshell.v1.RejectDraftChunkResponse - (*DraftChunkApproval)(nil), // 215: openshell.v1.DraftChunkApproval - (*ApproveAllDraftChunksRequest)(nil), // 216: openshell.v1.ApproveAllDraftChunksRequest - (*ApproveAllDraftChunksResponse)(nil), // 217: openshell.v1.ApproveAllDraftChunksResponse - (*EditDraftChunkRequest)(nil), // 218: openshell.v1.EditDraftChunkRequest - (*EditDraftChunkResponse)(nil), // 219: openshell.v1.EditDraftChunkResponse - (*UndoDraftChunkRequest)(nil), // 220: openshell.v1.UndoDraftChunkRequest - (*UndoDraftChunkResponse)(nil), // 221: openshell.v1.UndoDraftChunkResponse - (*ClearDraftChunksRequest)(nil), // 222: openshell.v1.ClearDraftChunksRequest - (*ClearDraftChunksResponse)(nil), // 223: openshell.v1.ClearDraftChunksResponse - (*GetDraftHistoryRequest)(nil), // 224: openshell.v1.GetDraftHistoryRequest - (*DraftHistoryEntry)(nil), // 225: openshell.v1.DraftHistoryEntry - (*GetDraftHistoryResponse)(nil), // 226: openshell.v1.GetDraftHistoryResponse - (*CreateWorkspaceRequest)(nil), // 227: openshell.v1.CreateWorkspaceRequest - (*CreateWorkspaceResponse)(nil), // 228: openshell.v1.CreateWorkspaceResponse - (*GetWorkspaceRequest)(nil), // 229: openshell.v1.GetWorkspaceRequest - (*GetWorkspaceResponse)(nil), // 230: openshell.v1.GetWorkspaceResponse - (*ListWorkspacesRequest)(nil), // 231: openshell.v1.ListWorkspacesRequest - (*ListWorkspacesResponse)(nil), // 232: openshell.v1.ListWorkspacesResponse - (*DeleteWorkspaceRequest)(nil), // 233: openshell.v1.DeleteWorkspaceRequest - (*DeleteWorkspaceResponse)(nil), // 234: openshell.v1.DeleteWorkspaceResponse - (*WorkspaceMember)(nil), // 235: openshell.v1.WorkspaceMember - (*AddWorkspaceMemberRequest)(nil), // 236: openshell.v1.AddWorkspaceMemberRequest - (*AddWorkspaceMemberResponse)(nil), // 237: openshell.v1.AddWorkspaceMemberResponse - (*RemoveWorkspaceMemberRequest)(nil), // 238: openshell.v1.RemoveWorkspaceMemberRequest - (*RemoveWorkspaceMemberResponse)(nil), // 239: openshell.v1.RemoveWorkspaceMemberResponse - (*ListWorkspaceMembersRequest)(nil), // 240: openshell.v1.ListWorkspaceMembersRequest - (*ListWorkspaceMembersResponse)(nil), // 241: openshell.v1.ListWorkspaceMembersResponse - (*ExtensionServiceCredential)(nil), // 242: openshell.v1.ExtensionServiceCredential - (*EndpointObservation)(nil), // 243: openshell.v1.EndpointObservation - (*ReportEndpointStatusRequest)(nil), // 244: openshell.v1.ReportEndpointStatusRequest - (*ReportEndpointStatusResponse)(nil), // 245: openshell.v1.ReportEndpointStatusResponse - (*EndpointStatus)(nil), // 246: openshell.v1.EndpointStatus - (*SandboxProvisioning)(nil), // 247: openshell.v1.SandboxProvisioning - (*SandboxServiceExposure)(nil), // 248: openshell.v1.SandboxServiceExposure - nil, // 249: openshell.v1.SandboxSpec.EnvironmentEntry - nil, // 250: openshell.v1.SandboxTemplate.LabelsEntry - nil, // 251: openshell.v1.SandboxTemplate.AnnotationsEntry - nil, // 252: openshell.v1.SandboxTemplate.EnvironmentEntry - nil, // 253: openshell.v1.SandboxWorkloadConfig.EnvironmentEntry - nil, // 254: openshell.v1.PlatformEvent.MetadataEntry - nil, // 255: openshell.v1.CreateSandboxRequest.LabelsEntry - nil, // 256: openshell.v1.CreateSandboxRequest.AnnotationsEntry - nil, // 257: openshell.v1.SandboxResponse.ServiceUrlsEntry - nil, // 258: openshell.v1.ExecSandboxRequest.EnvironmentEntry - nil, // 259: openshell.v1.SandboxLogLine.FieldsEntry - nil, // 260: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry - nil, // 261: openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry - nil, // 262: openshell.v1.ProviderProfile.AnnotationsEntry - nil, // 263: openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry - nil, // 264: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry - nil, // 265: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry - nil, // 266: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry - nil, // 267: openshell.v1.UpdateConfigRequest.AnnotationsEntry - nil, // 268: openshell.v1.UpdateConfigResponse.AnnotationsEntry - nil, // 269: openshell.v1.SandboxPolicyRevision.ProvenanceEntry - nil, // 270: openshell.v1.CreateWorkspaceRequest.LabelsEntry - (*timestamppb.Timestamp)(nil), // 271: google.protobuf.Timestamp - (*datamodelv1.ObjectMeta)(nil), // 272: openshell.datamodel.v1.ObjectMeta - (*sandboxv1.SandboxPolicy)(nil), // 273: openshell.sandbox.v1.SandboxPolicy - (*structpb.Struct)(nil), // 274: google.protobuf.Struct - (*durationpb.Duration)(nil), // 275: google.protobuf.Duration - (*datamodelv1.WorkspaceSelector)(nil), // 276: openshell.datamodel.v1.WorkspaceSelector - (*datamodelv1.Provider)(nil), // 277: openshell.datamodel.v1.Provider - (sandboxv1.PolicySource)(0), // 278: openshell.sandbox.v1.PolicySource - (*sandboxv1.NetworkEndpoint)(nil), // 279: openshell.sandbox.v1.NetworkEndpoint - (*sandboxv1.NetworkBinary)(nil), // 280: openshell.sandbox.v1.NetworkBinary - (*sandboxv1.SettingValue)(nil), // 281: openshell.sandbox.v1.SettingValue - (*sandboxv1.NetworkPolicyRule)(nil), // 282: openshell.sandbox.v1.NetworkPolicyRule - (*sandboxv1.L7DenyRule)(nil), // 283: openshell.sandbox.v1.L7DenyRule - (*sandboxv1.L7Rule)(nil), // 284: openshell.sandbox.v1.L7Rule - (*datamodelv1.Workspace)(nil), // 285: openshell.datamodel.v1.Workspace - (*sandboxv1.GetSandboxConfigRequest)(nil), // 286: openshell.sandbox.v1.GetSandboxConfigRequest - (*sandboxv1.GetGatewayConfigRequest)(nil), // 287: openshell.sandbox.v1.GetGatewayConfigRequest - (*sandboxv1.GetSandboxConfigResponse)(nil), // 288: openshell.sandbox.v1.GetSandboxConfigResponse - (*sandboxv1.GetGatewayConfigResponse)(nil), // 289: openshell.sandbox.v1.GetGatewayConfigResponse + (SandboxRestartPolicy)(0), // 16: openshell.v1.SandboxRestartPolicy + (DeletionOutcome)(0), // 17: openshell.v1.DeletionOutcome + (EndpointResult)(0), // 18: openshell.v1.EndpointResult + (*IssueSandboxTokenRequest)(nil), // 19: openshell.v1.IssueSandboxTokenRequest + (*IssueSandboxTokenResponse)(nil), // 20: openshell.v1.IssueSandboxTokenResponse + (*RefreshSandboxTokenRequest)(nil), // 21: openshell.v1.RefreshSandboxTokenRequest + (*RefreshSandboxTokenResponse)(nil), // 22: openshell.v1.RefreshSandboxTokenResponse + (*HealthRequest)(nil), // 23: openshell.v1.HealthRequest + (*HealthResponse)(nil), // 24: openshell.v1.HealthResponse + (*GetCurrentUserRequest)(nil), // 25: openshell.v1.GetCurrentUserRequest + (*GetCurrentUserResponse)(nil), // 26: openshell.v1.GetCurrentUserResponse + (*GetGatewayInfoRequest)(nil), // 27: openshell.v1.GetGatewayInfoRequest + (*GetGatewayInfoResponse)(nil), // 28: openshell.v1.GetGatewayInfoResponse + (*NegotiatedExtensionInfo)(nil), // 29: openshell.v1.NegotiatedExtensionInfo + (*ComputeDriverInfo)(nil), // 30: openshell.v1.ComputeDriverInfo + (*ComputeDriverCapabilities)(nil), // 31: openshell.v1.ComputeDriverCapabilities + (*ResourceCapabilities)(nil), // 32: openshell.v1.ResourceCapabilities + (*CpuResourceCapabilities)(nil), // 33: openshell.v1.CpuResourceCapabilities + (*MemoryResourceCapabilities)(nil), // 34: openshell.v1.MemoryResourceCapabilities + (*GpuResourceCapabilities)(nil), // 35: openshell.v1.GpuResourceCapabilities + (*Sandbox)(nil), // 36: openshell.v1.Sandbox + (*SandboxSpec)(nil), // 37: openshell.v1.SandboxSpec + (*ResourceRequirements)(nil), // 38: openshell.v1.ResourceRequirements + (*GpuResourceRequirements)(nil), // 39: openshell.v1.GpuResourceRequirements + (*SandboxTemplate)(nil), // 40: openshell.v1.SandboxTemplate + (*SandboxWorkloadTemplate)(nil), // 41: openshell.v1.SandboxWorkloadTemplate + (*SandboxWorkloadTemplateSpec)(nil), // 42: openshell.v1.SandboxWorkloadTemplateSpec + (*SandboxWorkloadConfig)(nil), // 43: openshell.v1.SandboxWorkloadConfig + (*SandboxResources)(nil), // 44: openshell.v1.SandboxResources + (*SandboxServiceLevel)(nil), // 45: openshell.v1.SandboxServiceLevel + (*SandboxStartup)(nil), // 46: openshell.v1.SandboxStartup + (*SandboxWorkloadTemplateProvenance)(nil), // 47: openshell.v1.SandboxWorkloadTemplateProvenance + (*SandboxStatus)(nil), // 48: openshell.v1.SandboxStatus + (*SandboxCondition)(nil), // 49: openshell.v1.SandboxCondition + (*PlatformEvent)(nil), // 50: openshell.v1.PlatformEvent + (*CreateSandboxRequest)(nil), // 51: openshell.v1.CreateSandboxRequest + (*CreateSandboxTemplateRequest)(nil), // 52: openshell.v1.CreateSandboxTemplateRequest + (*GetSandboxTemplateRequest)(nil), // 53: openshell.v1.GetSandboxTemplateRequest + (*ListSandboxTemplatesRequest)(nil), // 54: openshell.v1.ListSandboxTemplatesRequest + (*DeleteSandboxTemplateRequest)(nil), // 55: openshell.v1.DeleteSandboxTemplateRequest + (*SandboxTemplateResponse)(nil), // 56: openshell.v1.SandboxTemplateResponse + (*ListSandboxTemplatesResponse)(nil), // 57: openshell.v1.ListSandboxTemplatesResponse + (*DeleteSandboxTemplateResponse)(nil), // 58: openshell.v1.DeleteSandboxTemplateResponse + (*BeginRootfsTarStagingRequest)(nil), // 59: openshell.v1.BeginRootfsTarStagingRequest + (*BeginRootfsTarStagingResponse)(nil), // 60: openshell.v1.BeginRootfsTarStagingResponse + (*GetSandboxRequest)(nil), // 61: openshell.v1.GetSandboxRequest + (*ListSandboxesRequest)(nil), // 62: openshell.v1.ListSandboxesRequest + (*ListSandboxProvidersRequest)(nil), // 63: openshell.v1.ListSandboxProvidersRequest + (*AttachSandboxProviderRequest)(nil), // 64: openshell.v1.AttachSandboxProviderRequest + (*DetachSandboxProviderRequest)(nil), // 65: openshell.v1.DetachSandboxProviderRequest + (*DeleteSandboxRequest)(nil), // 66: openshell.v1.DeleteSandboxRequest + (*StopSandboxRequest)(nil), // 67: openshell.v1.StopSandboxRequest + (*StartSandboxRequest)(nil), // 68: openshell.v1.StartSandboxRequest + (*SandboxResponse)(nil), // 69: openshell.v1.SandboxResponse + (*ListSandboxesResponse)(nil), // 70: openshell.v1.ListSandboxesResponse + (*ListSandboxProvidersResponse)(nil), // 71: openshell.v1.ListSandboxProvidersResponse + (*AttachSandboxProviderResponse)(nil), // 72: openshell.v1.AttachSandboxProviderResponse + (*DetachSandboxProviderResponse)(nil), // 73: openshell.v1.DetachSandboxProviderResponse + (*ProviderDesiredIdentity)(nil), // 74: openshell.v1.ProviderDesiredIdentity + (*ConfigSnapshotRevision)(nil), // 75: openshell.v1.ConfigSnapshotRevision + (*SandboxConfigRevision)(nil), // 76: openshell.v1.SandboxConfigRevision + (*ConfigUpdateOperation)(nil), // 77: openshell.v1.ConfigUpdateOperation + (*ProviderMutationReceipt)(nil), // 78: openshell.v1.ProviderMutationReceipt + (*ProviderReadinessObservation)(nil), // 79: openshell.v1.ProviderReadinessObservation + (*ProviderReadinessStatus)(nil), // 80: openshell.v1.ProviderReadinessStatus + (*GetSandboxProviderStatusRequest)(nil), // 81: openshell.v1.GetSandboxProviderStatusRequest + (*GetSandboxProviderStatusResponse)(nil), // 82: openshell.v1.GetSandboxProviderStatusResponse + (*ReportProviderReadinessRequest)(nil), // 83: openshell.v1.ReportProviderReadinessRequest + (*ReportProviderReadinessResponse)(nil), // 84: openshell.v1.ReportProviderReadinessResponse + (*DeleteSandboxResponse)(nil), // 85: openshell.v1.DeleteSandboxResponse + (*CreateSshSessionRequest)(nil), // 86: openshell.v1.CreateSshSessionRequest + (*CreateSshSessionResponse)(nil), // 87: openshell.v1.CreateSshSessionResponse + (*ExposeServiceRequest)(nil), // 88: openshell.v1.ExposeServiceRequest + (*GetServiceRequest)(nil), // 89: openshell.v1.GetServiceRequest + (*ListServicesRequest)(nil), // 90: openshell.v1.ListServicesRequest + (*ListServicesResponse)(nil), // 91: openshell.v1.ListServicesResponse + (*DeleteServiceRequest)(nil), // 92: openshell.v1.DeleteServiceRequest + (*DeleteServiceResponse)(nil), // 93: openshell.v1.DeleteServiceResponse + (*ServiceEndpoint)(nil), // 94: openshell.v1.ServiceEndpoint + (*ServiceEndpointResponse)(nil), // 95: openshell.v1.ServiceEndpointResponse + (*RevokeSshSessionRequest)(nil), // 96: openshell.v1.RevokeSshSessionRequest + (*RevokeSshSessionResponse)(nil), // 97: openshell.v1.RevokeSshSessionResponse + (*ExecSandboxRequest)(nil), // 98: openshell.v1.ExecSandboxRequest + (*ExecSandboxStdout)(nil), // 99: openshell.v1.ExecSandboxStdout + (*ExecSandboxStderr)(nil), // 100: openshell.v1.ExecSandboxStderr + (*ExecSandboxExit)(nil), // 101: openshell.v1.ExecSandboxExit + (*ExecSandboxEvent)(nil), // 102: openshell.v1.ExecSandboxEvent + (*TcpForwardInit)(nil), // 103: openshell.v1.TcpForwardInit + (*TcpForwardFrame)(nil), // 104: openshell.v1.TcpForwardFrame + (*ExecSandboxInput)(nil), // 105: openshell.v1.ExecSandboxInput + (*ExecSandboxWindowResize)(nil), // 106: openshell.v1.ExecSandboxWindowResize + (*SshSession)(nil), // 107: openshell.v1.SshSession + (*WatchSandboxRequest)(nil), // 108: openshell.v1.WatchSandboxRequest + (*SandboxStreamEvent)(nil), // 109: openshell.v1.SandboxStreamEvent + (*SandboxLogLine)(nil), // 110: openshell.v1.SandboxLogLine + (*SandboxStreamWarning)(nil), // 111: openshell.v1.SandboxStreamWarning + (*CreateProviderRequest)(nil), // 112: openshell.v1.CreateProviderRequest + (*GetProviderRequest)(nil), // 113: openshell.v1.GetProviderRequest + (*ListProvidersRequest)(nil), // 114: openshell.v1.ListProvidersRequest + (*UpdateProviderRequest)(nil), // 115: openshell.v1.UpdateProviderRequest + (*DeleteProviderRequest)(nil), // 116: openshell.v1.DeleteProviderRequest + (*ProviderResponse)(nil), // 117: openshell.v1.ProviderResponse + (*ListProvidersResponse)(nil), // 118: openshell.v1.ListProvidersResponse + (*ListProviderProfilesRequest)(nil), // 119: openshell.v1.ListProviderProfilesRequest + (*GetProviderProfileRequest)(nil), // 120: openshell.v1.GetProviderProfileRequest + (*ProviderProfileImportItem)(nil), // 121: openshell.v1.ProviderProfileImportItem + (*ProviderProfileDiagnostic)(nil), // 122: openshell.v1.ProviderProfileDiagnostic + (*ProviderCredentialTokenGrantAudienceOverride)(nil), // 123: openshell.v1.ProviderCredentialTokenGrantAudienceOverride + (*ProviderCredentialTokenGrantSubjectToken)(nil), // 124: openshell.v1.ProviderCredentialTokenGrantSubjectToken + (*ProviderCredentialTokenGrant)(nil), // 125: openshell.v1.ProviderCredentialTokenGrant + (*ProviderProfileCredential)(nil), // 126: openshell.v1.ProviderProfileCredential + (*ProviderCredentialRefreshMaterial)(nil), // 127: openshell.v1.ProviderCredentialRefreshMaterial + (*ProviderCredentialRefreshOutput)(nil), // 128: openshell.v1.ProviderCredentialRefreshOutput + (*ProviderCredentialRefresh)(nil), // 129: openshell.v1.ProviderCredentialRefresh + (*ProviderCredentialRefreshStatus)(nil), // 130: openshell.v1.ProviderCredentialRefreshStatus + (*ProviderProfileDiscovery)(nil), // 131: openshell.v1.ProviderProfileDiscovery + (*GetProviderRefreshStatusRequest)(nil), // 132: openshell.v1.GetProviderRefreshStatusRequest + (*GetProviderRefreshStatusResponse)(nil), // 133: openshell.v1.GetProviderRefreshStatusResponse + (*ConfigureProviderRefreshRequest)(nil), // 134: openshell.v1.ConfigureProviderRefreshRequest + (*ConfigureProviderRefreshResponse)(nil), // 135: openshell.v1.ConfigureProviderRefreshResponse + (*RotateProviderCredentialRequest)(nil), // 136: openshell.v1.RotateProviderCredentialRequest + (*RotateProviderCredentialResponse)(nil), // 137: openshell.v1.RotateProviderCredentialResponse + (*DeleteProviderRefreshRequest)(nil), // 138: openshell.v1.DeleteProviderRefreshRequest + (*DeleteProviderRefreshResponse)(nil), // 139: openshell.v1.DeleteProviderRefreshResponse + (*ProviderProfile)(nil), // 140: openshell.v1.ProviderProfile + (*ProviderProfileResponse)(nil), // 141: openshell.v1.ProviderProfileResponse + (*ListProviderProfilesResponse)(nil), // 142: openshell.v1.ListProviderProfilesResponse + (*ImportProviderProfilesRequest)(nil), // 143: openshell.v1.ImportProviderProfilesRequest + (*ImportProviderProfilesResponse)(nil), // 144: openshell.v1.ImportProviderProfilesResponse + (*UpdateProviderProfilesRequest)(nil), // 145: openshell.v1.UpdateProviderProfilesRequest + (*UpdateProviderProfilesResponse)(nil), // 146: openshell.v1.UpdateProviderProfilesResponse + (*LintProviderProfilesRequest)(nil), // 147: openshell.v1.LintProviderProfilesRequest + (*LintProviderProfilesResponse)(nil), // 148: openshell.v1.LintProviderProfilesResponse + (*DeleteProviderResponse)(nil), // 149: openshell.v1.DeleteProviderResponse + (*DeleteProviderProfileRequest)(nil), // 150: openshell.v1.DeleteProviderProfileRequest + (*DeleteProviderProfileResponse)(nil), // 151: openshell.v1.DeleteProviderProfileResponse + (*GetSandboxProviderEnvironmentRequest)(nil), // 152: openshell.v1.GetSandboxProviderEnvironmentRequest + (*StaticCredentialEndpointBinding)(nil), // 153: openshell.v1.StaticCredentialEndpointBinding + (*StaticCredentialBinding)(nil), // 154: openshell.v1.StaticCredentialBinding + (*GetSandboxProviderEnvironmentResponse)(nil), // 155: openshell.v1.GetSandboxProviderEnvironmentResponse + (*ExchangeProviderSubjectTokenRequest)(nil), // 156: openshell.v1.ExchangeProviderSubjectTokenRequest + (*ExchangeProviderSubjectTokenResponse)(nil), // 157: openshell.v1.ExchangeProviderSubjectTokenResponse + (*UpdateConfigRequest)(nil), // 158: openshell.v1.UpdateConfigRequest + (*PolicyMergeOperation)(nil), // 159: openshell.v1.PolicyMergeOperation + (*AddNetworkRule)(nil), // 160: openshell.v1.AddNetworkRule + (*RemoveNetworkEndpoint)(nil), // 161: openshell.v1.RemoveNetworkEndpoint + (*RemoveNetworkRule)(nil), // 162: openshell.v1.RemoveNetworkRule + (*L7RuleTarget)(nil), // 163: openshell.v1.L7RuleTarget + (*AddDenyRules)(nil), // 164: openshell.v1.AddDenyRules + (*AddAllowRules)(nil), // 165: openshell.v1.AddAllowRules + (*RemoveNetworkBinary)(nil), // 166: openshell.v1.RemoveNetworkBinary + (*UpdateConfigResponse)(nil), // 167: openshell.v1.UpdateConfigResponse + (*GetSandboxPolicyStatusRequest)(nil), // 168: openshell.v1.GetSandboxPolicyStatusRequest + (*GetSandboxPolicyStatusResponse)(nil), // 169: openshell.v1.GetSandboxPolicyStatusResponse + (*ListSandboxPoliciesRequest)(nil), // 170: openshell.v1.ListSandboxPoliciesRequest + (*ListSandboxPoliciesResponse)(nil), // 171: openshell.v1.ListSandboxPoliciesResponse + (*ReportPolicyStatusRequest)(nil), // 172: openshell.v1.ReportPolicyStatusRequest + (*ReportPolicyStatusResponse)(nil), // 173: openshell.v1.ReportPolicyStatusResponse + (*SandboxConfigurationAdmission)(nil), // 174: openshell.v1.SandboxConfigurationAdmission + (*ReportSandboxConfigurationRequest)(nil), // 175: openshell.v1.ReportSandboxConfigurationRequest + (*ReportSandboxConfigurationResponse)(nil), // 176: openshell.v1.ReportSandboxConfigurationResponse + (*SandboxPolicyRevision)(nil), // 177: openshell.v1.SandboxPolicyRevision + (*GetSandboxLogsRequest)(nil), // 178: openshell.v1.GetSandboxLogsRequest + (*PushSandboxLogsRequest)(nil), // 179: openshell.v1.PushSandboxLogsRequest + (*PushSandboxLogsResponse)(nil), // 180: openshell.v1.PushSandboxLogsResponse + (*GetSandboxLogsResponse)(nil), // 181: openshell.v1.GetSandboxLogsResponse + (*SupervisorMessage)(nil), // 182: openshell.v1.SupervisorMessage + (*GatewayMessage)(nil), // 183: openshell.v1.GatewayMessage + (*SupervisorHello)(nil), // 184: openshell.v1.SupervisorHello + (*SessionAccepted)(nil), // 185: openshell.v1.SessionAccepted + (*SessionRejected)(nil), // 186: openshell.v1.SessionRejected + (*SupervisorHeartbeat)(nil), // 187: openshell.v1.SupervisorHeartbeat + (*GatewayHeartbeat)(nil), // 188: openshell.v1.GatewayHeartbeat + (*ReportMainProcessExitRequest)(nil), // 189: openshell.v1.ReportMainProcessExitRequest + (*ReportMainProcessExitResponse)(nil), // 190: openshell.v1.ReportMainProcessExitResponse + (*FinalizeMainProcessExitRequest)(nil), // 191: openshell.v1.FinalizeMainProcessExitRequest + (*FinalizeMainProcessExitResponse)(nil), // 192: openshell.v1.FinalizeMainProcessExitResponse + (*RelayOpen)(nil), // 193: openshell.v1.RelayOpen + (*SshRelayTarget)(nil), // 194: openshell.v1.SshRelayTarget + (*TcpRelayTarget)(nil), // 195: openshell.v1.TcpRelayTarget + (*RelayInit)(nil), // 196: openshell.v1.RelayInit + (*RelayFrame)(nil), // 197: openshell.v1.RelayFrame + (*PeerRelayInit)(nil), // 198: openshell.v1.PeerRelayInit + (*PeerRelayFrame)(nil), // 199: openshell.v1.PeerRelayFrame + (*RelayOpenResult)(nil), // 200: openshell.v1.RelayOpenResult + (*RelayClose)(nil), // 201: openshell.v1.RelayClose + (*L7RequestSample)(nil), // 202: openshell.v1.L7RequestSample + (*DenialSummary)(nil), // 203: openshell.v1.DenialSummary + (*DenialGroupCount)(nil), // 204: openshell.v1.DenialGroupCount + (*NetworkActivitySummary)(nil), // 205: openshell.v1.NetworkActivitySummary + (*PolicyChunk)(nil), // 206: openshell.v1.PolicyChunk + (*DraftPolicyUpdate)(nil), // 207: openshell.v1.DraftPolicyUpdate + (*SubmitPolicyAnalysisRequest)(nil), // 208: openshell.v1.SubmitPolicyAnalysisRequest + (*SubmitPolicyAnalysisResponse)(nil), // 209: openshell.v1.SubmitPolicyAnalysisResponse + (*GetDraftPolicyRequest)(nil), // 210: openshell.v1.GetDraftPolicyRequest + (*GetDraftPolicyResponse)(nil), // 211: openshell.v1.GetDraftPolicyResponse + (*ApproveDraftChunkRequest)(nil), // 212: openshell.v1.ApproveDraftChunkRequest + (*ApproveDraftChunkResponse)(nil), // 213: openshell.v1.ApproveDraftChunkResponse + (*RejectDraftChunkRequest)(nil), // 214: openshell.v1.RejectDraftChunkRequest + (*RejectDraftChunkResponse)(nil), // 215: openshell.v1.RejectDraftChunkResponse + (*DraftChunkApproval)(nil), // 216: openshell.v1.DraftChunkApproval + (*ApproveAllDraftChunksRequest)(nil), // 217: openshell.v1.ApproveAllDraftChunksRequest + (*ApproveAllDraftChunksResponse)(nil), // 218: openshell.v1.ApproveAllDraftChunksResponse + (*EditDraftChunkRequest)(nil), // 219: openshell.v1.EditDraftChunkRequest + (*EditDraftChunkResponse)(nil), // 220: openshell.v1.EditDraftChunkResponse + (*UndoDraftChunkRequest)(nil), // 221: openshell.v1.UndoDraftChunkRequest + (*UndoDraftChunkResponse)(nil), // 222: openshell.v1.UndoDraftChunkResponse + (*ClearDraftChunksRequest)(nil), // 223: openshell.v1.ClearDraftChunksRequest + (*ClearDraftChunksResponse)(nil), // 224: openshell.v1.ClearDraftChunksResponse + (*GetDraftHistoryRequest)(nil), // 225: openshell.v1.GetDraftHistoryRequest + (*DraftHistoryEntry)(nil), // 226: openshell.v1.DraftHistoryEntry + (*GetDraftHistoryResponse)(nil), // 227: openshell.v1.GetDraftHistoryResponse + (*CreateWorkspaceRequest)(nil), // 228: openshell.v1.CreateWorkspaceRequest + (*CreateWorkspaceResponse)(nil), // 229: openshell.v1.CreateWorkspaceResponse + (*GetWorkspaceRequest)(nil), // 230: openshell.v1.GetWorkspaceRequest + (*GetWorkspaceResponse)(nil), // 231: openshell.v1.GetWorkspaceResponse + (*ListWorkspacesRequest)(nil), // 232: openshell.v1.ListWorkspacesRequest + (*ListWorkspacesResponse)(nil), // 233: openshell.v1.ListWorkspacesResponse + (*DeleteWorkspaceRequest)(nil), // 234: openshell.v1.DeleteWorkspaceRequest + (*DeleteWorkspaceResponse)(nil), // 235: openshell.v1.DeleteWorkspaceResponse + (*WorkspaceMember)(nil), // 236: openshell.v1.WorkspaceMember + (*AddWorkspaceMemberRequest)(nil), // 237: openshell.v1.AddWorkspaceMemberRequest + (*AddWorkspaceMemberResponse)(nil), // 238: openshell.v1.AddWorkspaceMemberResponse + (*RemoveWorkspaceMemberRequest)(nil), // 239: openshell.v1.RemoveWorkspaceMemberRequest + (*RemoveWorkspaceMemberResponse)(nil), // 240: openshell.v1.RemoveWorkspaceMemberResponse + (*ListWorkspaceMembersRequest)(nil), // 241: openshell.v1.ListWorkspaceMembersRequest + (*ListWorkspaceMembersResponse)(nil), // 242: openshell.v1.ListWorkspaceMembersResponse + (*ExtensionServiceCredential)(nil), // 243: openshell.v1.ExtensionServiceCredential + (*EndpointObservation)(nil), // 244: openshell.v1.EndpointObservation + (*ReportEndpointStatusRequest)(nil), // 245: openshell.v1.ReportEndpointStatusRequest + (*ReportEndpointStatusResponse)(nil), // 246: openshell.v1.ReportEndpointStatusResponse + (*EndpointStatus)(nil), // 247: openshell.v1.EndpointStatus + (*SandboxProvisioning)(nil), // 248: openshell.v1.SandboxProvisioning + (*SandboxServiceExposure)(nil), // 249: openshell.v1.SandboxServiceExposure + nil, // 250: openshell.v1.SandboxSpec.EnvironmentEntry + nil, // 251: openshell.v1.SandboxTemplate.LabelsEntry + nil, // 252: openshell.v1.SandboxTemplate.AnnotationsEntry + nil, // 253: openshell.v1.SandboxTemplate.EnvironmentEntry + nil, // 254: openshell.v1.SandboxWorkloadConfig.EnvironmentEntry + nil, // 255: openshell.v1.PlatformEvent.MetadataEntry + nil, // 256: openshell.v1.CreateSandboxRequest.LabelsEntry + nil, // 257: openshell.v1.CreateSandboxRequest.AnnotationsEntry + nil, // 258: openshell.v1.SandboxResponse.ServiceUrlsEntry + nil, // 259: openshell.v1.ExecSandboxRequest.EnvironmentEntry + nil, // 260: openshell.v1.SandboxLogLine.FieldsEntry + nil, // 261: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry + nil, // 262: openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry + nil, // 263: openshell.v1.ProviderProfile.AnnotationsEntry + nil, // 264: openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry + nil, // 265: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry + nil, // 266: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry + nil, // 267: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry + nil, // 268: openshell.v1.UpdateConfigRequest.AnnotationsEntry + nil, // 269: openshell.v1.UpdateConfigResponse.AnnotationsEntry + nil, // 270: openshell.v1.SandboxPolicyRevision.ProvenanceEntry + nil, // 271: openshell.v1.CreateWorkspaceRequest.LabelsEntry + (*timestamppb.Timestamp)(nil), // 272: google.protobuf.Timestamp + (*datamodelv1.ObjectMeta)(nil), // 273: openshell.datamodel.v1.ObjectMeta + (*sandboxv1.SandboxPolicy)(nil), // 274: openshell.sandbox.v1.SandboxPolicy + (*structpb.Struct)(nil), // 275: google.protobuf.Struct + (*durationpb.Duration)(nil), // 276: google.protobuf.Duration + (*datamodelv1.WorkspaceSelector)(nil), // 277: openshell.datamodel.v1.WorkspaceSelector + (*datamodelv1.Provider)(nil), // 278: openshell.datamodel.v1.Provider + (sandboxv1.PolicySource)(0), // 279: openshell.sandbox.v1.PolicySource + (*sandboxv1.NetworkEndpoint)(nil), // 280: openshell.sandbox.v1.NetworkEndpoint + (*sandboxv1.NetworkBinary)(nil), // 281: openshell.sandbox.v1.NetworkBinary + (*sandboxv1.SettingValue)(nil), // 282: openshell.sandbox.v1.SettingValue + (*sandboxv1.NetworkPolicyRule)(nil), // 283: openshell.sandbox.v1.NetworkPolicyRule + (*sandboxv1.L7DenyRule)(nil), // 284: openshell.sandbox.v1.L7DenyRule + (*sandboxv1.L7Rule)(nil), // 285: openshell.sandbox.v1.L7Rule + (*datamodelv1.Workspace)(nil), // 286: openshell.datamodel.v1.Workspace + (*sandboxv1.GetSandboxConfigRequest)(nil), // 287: openshell.sandbox.v1.GetSandboxConfigRequest + (*sandboxv1.GetGatewayConfigRequest)(nil), // 288: openshell.sandbox.v1.GetGatewayConfigRequest + (*sandboxv1.GetSandboxConfigResponse)(nil), // 289: openshell.sandbox.v1.GetSandboxConfigResponse + (*sandboxv1.GetGatewayConfigResponse)(nil), // 290: openshell.sandbox.v1.GetGatewayConfigResponse } var file_openshell_proto_depIdxs = []int32{ - 271, // 0: openshell.v1.IssueSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp - 271, // 1: openshell.v1.RefreshSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp - 242, // 2: openshell.v1.RefreshSandboxTokenResponse.extension_credentials:type_name -> openshell.v1.ExtensionServiceCredential - 271, // 3: openshell.v1.RefreshSandboxTokenResponse.sandbox_expiration_time:type_name -> google.protobuf.Timestamp + 272, // 0: openshell.v1.IssueSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp + 272, // 1: openshell.v1.RefreshSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp + 243, // 2: openshell.v1.RefreshSandboxTokenResponse.extension_credentials:type_name -> openshell.v1.ExtensionServiceCredential + 272, // 3: openshell.v1.RefreshSandboxTokenResponse.sandbox_expiration_time:type_name -> google.protobuf.Timestamp 13, // 4: openshell.v1.HealthResponse.status:type_name -> openshell.v1.ServiceStatus 13, // 5: openshell.v1.GetGatewayInfoResponse.status:type_name -> openshell.v1.ServiceStatus - 29, // 6: openshell.v1.GetGatewayInfoResponse.compute_drivers:type_name -> openshell.v1.ComputeDriverInfo - 28, // 7: openshell.v1.GetGatewayInfoResponse.extensions:type_name -> openshell.v1.NegotiatedExtensionInfo + 30, // 6: openshell.v1.GetGatewayInfoResponse.compute_drivers:type_name -> openshell.v1.ComputeDriverInfo + 29, // 7: openshell.v1.GetGatewayInfoResponse.extensions:type_name -> openshell.v1.NegotiatedExtensionInfo 0, // 8: openshell.v1.NegotiatedExtensionInfo.kind:type_name -> openshell.v1.ExtensionKind - 30, // 9: openshell.v1.ComputeDriverInfo.capabilities:type_name -> openshell.v1.ComputeDriverCapabilities - 31, // 10: openshell.v1.ComputeDriverCapabilities.resource_capabilities:type_name -> openshell.v1.ResourceCapabilities - 32, // 11: openshell.v1.ResourceCapabilities.cpu:type_name -> openshell.v1.CpuResourceCapabilities - 33, // 12: openshell.v1.ResourceCapabilities.memory:type_name -> openshell.v1.MemoryResourceCapabilities - 34, // 13: openshell.v1.ResourceCapabilities.gpu:type_name -> openshell.v1.GpuResourceCapabilities - 272, // 14: openshell.v1.Sandbox.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 36, // 15: openshell.v1.Sandbox.spec:type_name -> openshell.v1.SandboxSpec - 47, // 16: openshell.v1.Sandbox.status:type_name -> openshell.v1.SandboxStatus - 46, // 17: openshell.v1.Sandbox.created_from_workload_template:type_name -> openshell.v1.SandboxWorkloadTemplateProvenance - 249, // 18: openshell.v1.SandboxSpec.environment:type_name -> openshell.v1.SandboxSpec.EnvironmentEntry - 39, // 19: openshell.v1.SandboxSpec.template:type_name -> openshell.v1.SandboxTemplate - 273, // 20: openshell.v1.SandboxSpec.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 37, // 21: openshell.v1.SandboxSpec.resource_requirements:type_name -> openshell.v1.ResourceRequirements - 38, // 22: openshell.v1.ResourceRequirements.gpu:type_name -> openshell.v1.GpuResourceRequirements - 250, // 23: openshell.v1.SandboxTemplate.labels:type_name -> openshell.v1.SandboxTemplate.LabelsEntry - 251, // 24: openshell.v1.SandboxTemplate.annotations:type_name -> openshell.v1.SandboxTemplate.AnnotationsEntry - 252, // 25: openshell.v1.SandboxTemplate.environment:type_name -> openshell.v1.SandboxTemplate.EnvironmentEntry - 274, // 26: openshell.v1.SandboxTemplate.resources:type_name -> google.protobuf.Struct - 274, // 27: openshell.v1.SandboxTemplate.driver_config:type_name -> google.protobuf.Struct - 272, // 28: openshell.v1.SandboxWorkloadTemplate.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 41, // 29: openshell.v1.SandboxWorkloadTemplate.spec:type_name -> openshell.v1.SandboxWorkloadTemplateSpec - 42, // 30: openshell.v1.SandboxWorkloadTemplateSpec.workload:type_name -> openshell.v1.SandboxWorkloadConfig - 274, // 31: openshell.v1.SandboxWorkloadTemplateSpec.driver_config:type_name -> google.protobuf.Struct - 44, // 32: openshell.v1.SandboxWorkloadTemplateSpec.desired_service_level:type_name -> openshell.v1.SandboxServiceLevel - 253, // 33: openshell.v1.SandboxWorkloadConfig.environment:type_name -> openshell.v1.SandboxWorkloadConfig.EnvironmentEntry - 43, // 34: openshell.v1.SandboxWorkloadConfig.resources:type_name -> openshell.v1.SandboxResources - 38, // 35: openshell.v1.SandboxResources.gpu:type_name -> openshell.v1.GpuResourceRequirements - 45, // 36: openshell.v1.SandboxServiceLevel.startup:type_name -> openshell.v1.SandboxStartup - 275, // 37: openshell.v1.SandboxStartup.ready_within:type_name -> google.protobuf.Duration - 48, // 38: openshell.v1.SandboxStatus.conditions:type_name -> openshell.v1.SandboxCondition - 1, // 39: openshell.v1.SandboxStatus.phase:type_name -> openshell.v1.SandboxPhase - 246, // 40: openshell.v1.SandboxStatus.endpoint_statuses:type_name -> openshell.v1.EndpointStatus - 173, // 41: openshell.v1.SandboxStatus.configuration_admission:type_name -> openshell.v1.SandboxConfigurationAdmission - 247, // 42: openshell.v1.SandboxStatus.provisioning:type_name -> openshell.v1.SandboxProvisioning - 271, // 43: openshell.v1.SandboxCondition.transition_time:type_name -> google.protobuf.Timestamp - 271, // 44: openshell.v1.PlatformEvent.event_time:type_name -> google.protobuf.Timestamp - 254, // 45: openshell.v1.PlatformEvent.metadata:type_name -> openshell.v1.PlatformEvent.MetadataEntry - 276, // 46: openshell.v1.CreateSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 36, // 47: openshell.v1.CreateSandboxRequest.spec:type_name -> openshell.v1.SandboxSpec - 255, // 48: openshell.v1.CreateSandboxRequest.labels:type_name -> openshell.v1.CreateSandboxRequest.LabelsEntry - 256, // 49: openshell.v1.CreateSandboxRequest.annotations:type_name -> openshell.v1.CreateSandboxRequest.AnnotationsEntry - 248, // 50: openshell.v1.CreateSandboxRequest.service_exposures:type_name -> openshell.v1.SandboxServiceExposure - 276, // 51: openshell.v1.CreateSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 40, // 52: openshell.v1.CreateSandboxTemplateRequest.template:type_name -> openshell.v1.SandboxWorkloadTemplate - 276, // 53: openshell.v1.GetSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 54: openshell.v1.ListSandboxTemplatesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 55: openshell.v1.DeleteSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 40, // 56: openshell.v1.SandboxTemplateResponse.template:type_name -> openshell.v1.SandboxWorkloadTemplate - 40, // 57: openshell.v1.ListSandboxTemplatesResponse.templates:type_name -> openshell.v1.SandboxWorkloadTemplate - 16, // 58: openshell.v1.DeleteSandboxTemplateResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 276, // 59: openshell.v1.BeginRootfsTarStagingRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 271, // 60: openshell.v1.BeginRootfsTarStagingResponse.expiration_time:type_name -> google.protobuf.Timestamp - 276, // 61: openshell.v1.GetSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 62: openshell.v1.ListSandboxesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 63: openshell.v1.ListSandboxProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 64: openshell.v1.AttachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 65: openshell.v1.DetachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 66: openshell.v1.DeleteSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 67: openshell.v1.StopSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 68: openshell.v1.StartSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 35, // 69: openshell.v1.SandboxResponse.sandbox:type_name -> openshell.v1.Sandbox - 257, // 70: openshell.v1.SandboxResponse.service_urls:type_name -> openshell.v1.SandboxResponse.ServiceUrlsEntry - 35, // 71: openshell.v1.ListSandboxesResponse.sandboxes:type_name -> openshell.v1.Sandbox - 277, // 72: openshell.v1.ListSandboxProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider - 35, // 73: openshell.v1.AttachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox - 77, // 74: openshell.v1.AttachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt - 35, // 75: openshell.v1.DetachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox - 77, // 76: openshell.v1.DetachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt - 75, // 77: openshell.v1.ConfigSnapshotRevision.sandbox_config:type_name -> openshell.v1.SandboxConfigRevision - 73, // 78: openshell.v1.ConfigSnapshotRevision.provider_target:type_name -> openshell.v1.ProviderDesiredIdentity - 278, // 79: openshell.v1.SandboxConfigRevision.policy_source:type_name -> openshell.sandbox.v1.PolicySource - 5, // 80: openshell.v1.ConfigUpdateOperation.component:type_name -> openshell.v1.ConfigComponent - 74, // 81: openshell.v1.ConfigUpdateOperation.target_revision:type_name -> openshell.v1.ConfigSnapshotRevision - 7, // 82: openshell.v1.ConfigUpdateOperation.state:type_name -> openshell.v1.ConfigUpdateOperationState - 6, // 83: openshell.v1.ConfigUpdateOperation.outcome:type_name -> openshell.v1.ConfigApplyOutcome - 271, // 84: openshell.v1.ConfigUpdateOperation.created_time:type_name -> google.protobuf.Timestamp - 271, // 85: openshell.v1.ConfigUpdateOperation.updated_time:type_name -> google.protobuf.Timestamp - 271, // 86: openshell.v1.ConfigUpdateOperation.completed_time:type_name -> google.protobuf.Timestamp - 2, // 87: openshell.v1.ProviderMutationReceipt.kind:type_name -> openshell.v1.ProviderMutationKind - 73, // 88: openshell.v1.ProviderMutationReceipt.desired:type_name -> openshell.v1.ProviderDesiredIdentity - 271, // 89: openshell.v1.ProviderMutationReceipt.persisted_time:type_name -> google.protobuf.Timestamp - 4, // 90: openshell.v1.ProviderReadinessObservation.reason:type_name -> openshell.v1.ProviderReadinessReason - 77, // 91: openshell.v1.ProviderReadinessStatus.receipt:type_name -> openshell.v1.ProviderMutationReceipt - 3, // 92: openshell.v1.ProviderReadinessStatus.state:type_name -> openshell.v1.ProviderReadinessState - 4, // 93: openshell.v1.ProviderReadinessStatus.reason:type_name -> openshell.v1.ProviderReadinessReason - 78, // 94: openshell.v1.ProviderReadinessStatus.observed:type_name -> openshell.v1.ProviderReadinessObservation - 271, // 95: openshell.v1.ProviderReadinessStatus.observed_time:type_name -> google.protobuf.Timestamp - 271, // 96: openshell.v1.ProviderReadinessStatus.evaluated_time:type_name -> google.protobuf.Timestamp - 76, // 97: openshell.v1.ProviderReadinessStatus.operation:type_name -> openshell.v1.ConfigUpdateOperation - 276, // 98: openshell.v1.GetSandboxProviderStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 79, // 99: openshell.v1.GetSandboxProviderStatusResponse.status:type_name -> openshell.v1.ProviderReadinessStatus - 78, // 100: openshell.v1.ReportProviderReadinessRequest.observation:type_name -> openshell.v1.ProviderReadinessObservation - 275, // 101: openshell.v1.ReportProviderReadinessResponse.report_interval:type_name -> google.protobuf.Duration - 275, // 102: openshell.v1.ReportProviderReadinessResponse.observation_ttl:type_name -> google.protobuf.Duration - 16, // 103: openshell.v1.DeleteSandboxResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 276, // 104: openshell.v1.CreateSshSessionRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 271, // 105: openshell.v1.CreateSshSessionResponse.expiration_time:type_name -> google.protobuf.Timestamp - 276, // 106: openshell.v1.ExposeServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 107: openshell.v1.GetServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 108: openshell.v1.ListServicesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 94, // 109: openshell.v1.ListServicesResponse.services:type_name -> openshell.v1.ServiceEndpointResponse - 276, // 110: openshell.v1.DeleteServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 16, // 111: openshell.v1.DeleteServiceResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 272, // 112: openshell.v1.ServiceEndpoint.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 93, // 113: openshell.v1.ServiceEndpointResponse.endpoint:type_name -> openshell.v1.ServiceEndpoint - 16, // 114: openshell.v1.RevokeSshSessionResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 276, // 115: openshell.v1.ExecSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 258, // 116: openshell.v1.ExecSandboxRequest.environment:type_name -> openshell.v1.ExecSandboxRequest.EnvironmentEntry - 275, // 117: openshell.v1.ExecSandboxRequest.execution_timeout:type_name -> google.protobuf.Duration - 98, // 118: openshell.v1.ExecSandboxEvent.stdout:type_name -> openshell.v1.ExecSandboxStdout - 99, // 119: openshell.v1.ExecSandboxEvent.stderr:type_name -> openshell.v1.ExecSandboxStderr - 100, // 120: openshell.v1.ExecSandboxEvent.exit:type_name -> openshell.v1.ExecSandboxExit - 193, // 121: openshell.v1.TcpForwardInit.ssh:type_name -> openshell.v1.SshRelayTarget - 194, // 122: openshell.v1.TcpForwardInit.tcp:type_name -> openshell.v1.TcpRelayTarget - 102, // 123: openshell.v1.TcpForwardFrame.init:type_name -> openshell.v1.TcpForwardInit - 97, // 124: openshell.v1.ExecSandboxInput.start:type_name -> openshell.v1.ExecSandboxRequest - 105, // 125: openshell.v1.ExecSandboxInput.resize:type_name -> openshell.v1.ExecSandboxWindowResize - 272, // 126: openshell.v1.SshSession.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 271, // 127: openshell.v1.SshSession.expiration_time:type_name -> google.protobuf.Timestamp - 276, // 128: openshell.v1.WatchSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 271, // 129: openshell.v1.WatchSandboxRequest.since_time:type_name -> google.protobuf.Timestamp - 35, // 130: openshell.v1.SandboxStreamEvent.sandbox:type_name -> openshell.v1.Sandbox - 109, // 131: openshell.v1.SandboxStreamEvent.log:type_name -> openshell.v1.SandboxLogLine - 49, // 132: openshell.v1.SandboxStreamEvent.event:type_name -> openshell.v1.PlatformEvent - 110, // 133: openshell.v1.SandboxStreamEvent.warning:type_name -> openshell.v1.SandboxStreamWarning - 206, // 134: openshell.v1.SandboxStreamEvent.draft_policy_update:type_name -> openshell.v1.DraftPolicyUpdate - 271, // 135: openshell.v1.SandboxLogLine.event_time:type_name -> google.protobuf.Timestamp - 259, // 136: openshell.v1.SandboxLogLine.fields:type_name -> openshell.v1.SandboxLogLine.FieldsEntry - 276, // 137: openshell.v1.CreateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 277, // 138: openshell.v1.CreateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider - 276, // 139: openshell.v1.GetProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 140: openshell.v1.ListProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 141: openshell.v1.UpdateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 277, // 142: openshell.v1.UpdateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider - 260, // 143: openshell.v1.UpdateProviderRequest.credential_expiration_times:type_name -> openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry - 276, // 144: openshell.v1.DeleteProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 277, // 145: openshell.v1.ProviderResponse.provider:type_name -> openshell.datamodel.v1.Provider - 77, // 146: openshell.v1.ProviderResponse.target_receipts:type_name -> openshell.v1.ProviderMutationReceipt - 277, // 147: openshell.v1.ListProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider - 276, // 148: openshell.v1.ListProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 149: openshell.v1.GetProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 139, // 150: openshell.v1.ProviderProfileImportItem.profile:type_name -> openshell.v1.ProviderProfile - 275, // 151: openshell.v1.ProviderCredentialTokenGrant.cache_ttl:type_name -> google.protobuf.Duration - 122, // 152: openshell.v1.ProviderCredentialTokenGrant.audience_overrides:type_name -> openshell.v1.ProviderCredentialTokenGrantAudienceOverride - 8, // 153: openshell.v1.ProviderCredentialTokenGrant.grant_type:type_name -> openshell.v1.ProviderCredentialTokenGrantType - 123, // 154: openshell.v1.ProviderCredentialTokenGrant.subject_token:type_name -> openshell.v1.ProviderCredentialTokenGrantSubjectToken - 128, // 155: openshell.v1.ProviderProfileCredential.refresh:type_name -> openshell.v1.ProviderCredentialRefresh - 124, // 156: openshell.v1.ProviderProfileCredential.token_grant:type_name -> openshell.v1.ProviderCredentialTokenGrant - 9, // 157: openshell.v1.ProviderCredentialRefresh.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 275, // 158: openshell.v1.ProviderCredentialRefresh.refresh_before:type_name -> google.protobuf.Duration - 275, // 159: openshell.v1.ProviderCredentialRefresh.max_lifetime:type_name -> google.protobuf.Duration - 126, // 160: openshell.v1.ProviderCredentialRefresh.material:type_name -> openshell.v1.ProviderCredentialRefreshMaterial - 127, // 161: openshell.v1.ProviderCredentialRefresh.additional_outputs:type_name -> openshell.v1.ProviderCredentialRefreshOutput - 9, // 162: openshell.v1.ProviderCredentialRefreshStatus.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 271, // 163: openshell.v1.ProviderCredentialRefreshStatus.expiration_time:type_name -> google.protobuf.Timestamp - 271, // 164: openshell.v1.ProviderCredentialRefreshStatus.next_refresh_time:type_name -> google.protobuf.Timestamp - 271, // 165: openshell.v1.ProviderCredentialRefreshStatus.last_refresh_time:type_name -> google.protobuf.Timestamp - 15, // 166: openshell.v1.ProviderCredentialRefreshStatus.recovery_action:type_name -> openshell.v1.ProviderCredentialRefreshRecoveryAction - 271, // 167: openshell.v1.ProviderCredentialRefreshStatus.last_error_time:type_name -> google.protobuf.Timestamp - 276, // 168: openshell.v1.GetProviderRefreshStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 129, // 169: openshell.v1.GetProviderRefreshStatusResponse.credentials:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 276, // 170: openshell.v1.ConfigureProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 9, // 171: openshell.v1.ConfigureProviderRefreshRequest.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 261, // 172: openshell.v1.ConfigureProviderRefreshRequest.material:type_name -> openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry - 271, // 173: openshell.v1.ConfigureProviderRefreshRequest.expiration_time:type_name -> google.protobuf.Timestamp - 129, // 174: openshell.v1.ConfigureProviderRefreshResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 276, // 175: openshell.v1.RotateProviderCredentialRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 129, // 176: openshell.v1.RotateProviderCredentialResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 276, // 177: openshell.v1.DeleteProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 16, // 178: openshell.v1.DeleteProviderRefreshResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 10, // 179: openshell.v1.ProviderProfile.category:type_name -> openshell.v1.ProviderProfileCategory - 125, // 180: openshell.v1.ProviderProfile.credentials:type_name -> openshell.v1.ProviderProfileCredential - 279, // 181: openshell.v1.ProviderProfile.endpoints:type_name -> openshell.sandbox.v1.NetworkEndpoint - 280, // 182: openshell.v1.ProviderProfile.binaries:type_name -> openshell.sandbox.v1.NetworkBinary - 130, // 183: openshell.v1.ProviderProfile.discovery:type_name -> openshell.v1.ProviderProfileDiscovery - 262, // 184: openshell.v1.ProviderProfile.annotations:type_name -> openshell.v1.ProviderProfile.AnnotationsEntry - 139, // 185: openshell.v1.ProviderProfileResponse.profile:type_name -> openshell.v1.ProviderProfile - 139, // 186: openshell.v1.ListProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile - 276, // 187: openshell.v1.ImportProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 120, // 188: openshell.v1.ImportProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem - 121, // 189: openshell.v1.ImportProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic - 139, // 190: openshell.v1.ImportProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile - 276, // 191: openshell.v1.UpdateProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 120, // 192: openshell.v1.UpdateProviderProfilesRequest.profile:type_name -> openshell.v1.ProviderProfileImportItem - 121, // 193: openshell.v1.UpdateProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic - 139, // 194: openshell.v1.UpdateProviderProfilesResponse.profile:type_name -> openshell.v1.ProviderProfile - 276, // 195: openshell.v1.LintProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 120, // 196: openshell.v1.LintProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem - 121, // 197: openshell.v1.LintProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic - 16, // 198: openshell.v1.DeleteProviderResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 276, // 199: openshell.v1.DeleteProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 16, // 200: openshell.v1.DeleteProviderProfileResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 152, // 201: openshell.v1.StaticCredentialBinding.endpoints:type_name -> openshell.v1.StaticCredentialEndpointBinding - 263, // 202: openshell.v1.GetSandboxProviderEnvironmentResponse.environment:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry - 264, // 203: openshell.v1.GetSandboxProviderEnvironmentResponse.credential_expiration_times:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry - 265, // 204: openshell.v1.GetSandboxProviderEnvironmentResponse.dynamic_credentials:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry - 266, // 205: openshell.v1.GetSandboxProviderEnvironmentResponse.static_credential_bindings:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry - 4, // 206: openshell.v1.GetSandboxProviderEnvironmentResponse.readiness_reason:type_name -> openshell.v1.ProviderReadinessReason - 275, // 207: openshell.v1.ExchangeProviderSubjectTokenResponse.expires_after:type_name -> google.protobuf.Duration - 276, // 208: openshell.v1.UpdateConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 273, // 209: openshell.v1.UpdateConfigRequest.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 281, // 210: openshell.v1.UpdateConfigRequest.setting_value:type_name -> openshell.sandbox.v1.SettingValue - 158, // 211: openshell.v1.UpdateConfigRequest.merge_operations:type_name -> openshell.v1.PolicyMergeOperation - 267, // 212: openshell.v1.UpdateConfigRequest.annotations:type_name -> openshell.v1.UpdateConfigRequest.AnnotationsEntry - 159, // 213: openshell.v1.PolicyMergeOperation.add_rule:type_name -> openshell.v1.AddNetworkRule - 160, // 214: openshell.v1.PolicyMergeOperation.remove_endpoint:type_name -> openshell.v1.RemoveNetworkEndpoint - 161, // 215: openshell.v1.PolicyMergeOperation.remove_rule:type_name -> openshell.v1.RemoveNetworkRule - 163, // 216: openshell.v1.PolicyMergeOperation.add_deny_rules:type_name -> openshell.v1.AddDenyRules - 164, // 217: openshell.v1.PolicyMergeOperation.add_allow_rules:type_name -> openshell.v1.AddAllowRules - 165, // 218: openshell.v1.PolicyMergeOperation.remove_binary:type_name -> openshell.v1.RemoveNetworkBinary - 282, // 219: openshell.v1.AddNetworkRule.rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 280, // 220: openshell.v1.L7RuleTarget.binaries:type_name -> openshell.sandbox.v1.NetworkBinary - 283, // 221: openshell.v1.AddDenyRules.deny_rules:type_name -> openshell.sandbox.v1.L7DenyRule - 162, // 222: openshell.v1.AddDenyRules.target:type_name -> openshell.v1.L7RuleTarget - 284, // 223: openshell.v1.AddAllowRules.rules:type_name -> openshell.sandbox.v1.L7Rule - 162, // 224: openshell.v1.AddAllowRules.target:type_name -> openshell.v1.L7RuleTarget - 268, // 225: openshell.v1.UpdateConfigResponse.annotations:type_name -> openshell.v1.UpdateConfigResponse.AnnotationsEntry - 276, // 226: openshell.v1.GetSandboxPolicyStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 176, // 227: openshell.v1.GetSandboxPolicyStatusResponse.revision:type_name -> openshell.v1.SandboxPolicyRevision - 276, // 228: openshell.v1.ListSandboxPoliciesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 176, // 229: openshell.v1.ListSandboxPoliciesResponse.revisions:type_name -> openshell.v1.SandboxPolicyRevision - 12, // 230: openshell.v1.ReportPolicyStatusRequest.status:type_name -> openshell.v1.PolicyStatus - 11, // 231: openshell.v1.SandboxConfigurationAdmission.state:type_name -> openshell.v1.ConfigurationAdmissionState - 173, // 232: openshell.v1.ReportSandboxConfigurationRequest.admission:type_name -> openshell.v1.SandboxConfigurationAdmission - 12, // 233: openshell.v1.SandboxPolicyRevision.status:type_name -> openshell.v1.PolicyStatus - 271, // 234: openshell.v1.SandboxPolicyRevision.created_time:type_name -> google.protobuf.Timestamp - 271, // 235: openshell.v1.SandboxPolicyRevision.loaded_time:type_name -> google.protobuf.Timestamp - 273, // 236: openshell.v1.SandboxPolicyRevision.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 269, // 237: openshell.v1.SandboxPolicyRevision.provenance:type_name -> openshell.v1.SandboxPolicyRevision.ProvenanceEntry - 276, // 238: openshell.v1.GetSandboxLogsRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 271, // 239: openshell.v1.GetSandboxLogsRequest.since_time:type_name -> google.protobuf.Timestamp - 109, // 240: openshell.v1.PushSandboxLogsRequest.logs:type_name -> openshell.v1.SandboxLogLine - 109, // 241: openshell.v1.GetSandboxLogsResponse.logs:type_name -> openshell.v1.SandboxLogLine - 183, // 242: openshell.v1.SupervisorMessage.hello:type_name -> openshell.v1.SupervisorHello - 186, // 243: openshell.v1.SupervisorMessage.heartbeat:type_name -> openshell.v1.SupervisorHeartbeat - 199, // 244: openshell.v1.SupervisorMessage.relay_open_result:type_name -> openshell.v1.RelayOpenResult - 200, // 245: openshell.v1.SupervisorMessage.relay_close:type_name -> openshell.v1.RelayClose - 184, // 246: openshell.v1.GatewayMessage.session_accepted:type_name -> openshell.v1.SessionAccepted - 185, // 247: openshell.v1.GatewayMessage.session_rejected:type_name -> openshell.v1.SessionRejected - 187, // 248: openshell.v1.GatewayMessage.heartbeat:type_name -> openshell.v1.GatewayHeartbeat - 192, // 249: openshell.v1.GatewayMessage.relay_open:type_name -> openshell.v1.RelayOpen - 200, // 250: openshell.v1.GatewayMessage.relay_close:type_name -> openshell.v1.RelayClose - 275, // 251: openshell.v1.SessionAccepted.heartbeat_interval:type_name -> google.protobuf.Duration - 193, // 252: openshell.v1.RelayOpen.ssh:type_name -> openshell.v1.SshRelayTarget - 194, // 253: openshell.v1.RelayOpen.tcp:type_name -> openshell.v1.TcpRelayTarget - 195, // 254: openshell.v1.RelayFrame.init:type_name -> openshell.v1.RelayInit - 192, // 255: openshell.v1.PeerRelayInit.relay_open:type_name -> openshell.v1.RelayOpen - 197, // 256: openshell.v1.PeerRelayFrame.init:type_name -> openshell.v1.PeerRelayInit - 271, // 257: openshell.v1.DenialSummary.first_seen_time:type_name -> google.protobuf.Timestamp - 271, // 258: openshell.v1.DenialSummary.last_seen_time:type_name -> google.protobuf.Timestamp - 201, // 259: openshell.v1.DenialSummary.l7_request_samples:type_name -> openshell.v1.L7RequestSample - 203, // 260: openshell.v1.NetworkActivitySummary.denials_by_group:type_name -> openshell.v1.DenialGroupCount - 282, // 261: openshell.v1.PolicyChunk.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 271, // 262: openshell.v1.PolicyChunk.created_time:type_name -> google.protobuf.Timestamp - 271, // 263: openshell.v1.PolicyChunk.decided_time:type_name -> google.protobuf.Timestamp - 271, // 264: openshell.v1.PolicyChunk.first_seen_time:type_name -> google.protobuf.Timestamp - 271, // 265: openshell.v1.PolicyChunk.last_seen_time:type_name -> google.protobuf.Timestamp - 273, // 266: openshell.v1.PolicyChunk.current_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 273, // 267: openshell.v1.PolicyChunk.candidate_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 276, // 268: openshell.v1.SubmitPolicyAnalysisRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 202, // 269: openshell.v1.SubmitPolicyAnalysisRequest.summaries:type_name -> openshell.v1.DenialSummary - 205, // 270: openshell.v1.SubmitPolicyAnalysisRequest.proposed_chunks:type_name -> openshell.v1.PolicyChunk - 204, // 271: openshell.v1.SubmitPolicyAnalysisRequest.network_activity_summaries:type_name -> openshell.v1.NetworkActivitySummary - 276, // 272: openshell.v1.GetDraftPolicyRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 205, // 273: openshell.v1.GetDraftPolicyResponse.chunks:type_name -> openshell.v1.PolicyChunk - 271, // 274: openshell.v1.GetDraftPolicyResponse.last_analyzed_time:type_name -> google.protobuf.Timestamp - 276, // 275: openshell.v1.ApproveDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 276: openshell.v1.RejectDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 277: openshell.v1.ApproveAllDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 215, // 278: openshell.v1.ApproveAllDraftChunksRequest.approvals:type_name -> openshell.v1.DraftChunkApproval - 276, // 279: openshell.v1.EditDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 282, // 280: openshell.v1.EditDraftChunkRequest.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 276, // 281: openshell.v1.UndoDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 282: openshell.v1.ClearDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 276, // 283: openshell.v1.GetDraftHistoryRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 271, // 284: openshell.v1.DraftHistoryEntry.event_time:type_name -> google.protobuf.Timestamp - 225, // 285: openshell.v1.GetDraftHistoryResponse.entries:type_name -> openshell.v1.DraftHistoryEntry - 270, // 286: openshell.v1.CreateWorkspaceRequest.labels:type_name -> openshell.v1.CreateWorkspaceRequest.LabelsEntry - 285, // 287: openshell.v1.CreateWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace - 285, // 288: openshell.v1.GetWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace - 285, // 289: openshell.v1.ListWorkspacesResponse.workspaces:type_name -> openshell.datamodel.v1.Workspace - 16, // 290: openshell.v1.DeleteWorkspaceResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 272, // 291: openshell.v1.WorkspaceMember.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 14, // 292: openshell.v1.WorkspaceMember.role:type_name -> openshell.v1.WorkspaceRole - 276, // 293: openshell.v1.AddWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 14, // 294: openshell.v1.AddWorkspaceMemberRequest.role:type_name -> openshell.v1.WorkspaceRole - 235, // 295: openshell.v1.AddWorkspaceMemberResponse.member:type_name -> openshell.v1.WorkspaceMember - 276, // 296: openshell.v1.RemoveWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 16, // 297: openshell.v1.RemoveWorkspaceMemberResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 276, // 298: openshell.v1.ListWorkspaceMembersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 235, // 299: openshell.v1.ListWorkspaceMembersResponse.members:type_name -> openshell.v1.WorkspaceMember - 271, // 300: openshell.v1.ExtensionServiceCredential.expiration_time:type_name -> google.protobuf.Timestamp - 17, // 301: openshell.v1.EndpointObservation.result:type_name -> openshell.v1.EndpointResult - 243, // 302: openshell.v1.ReportEndpointStatusRequest.observations:type_name -> openshell.v1.EndpointObservation - 17, // 303: openshell.v1.EndpointStatus.last_result:type_name -> openshell.v1.EndpointResult - 271, // 304: openshell.v1.EndpointStatus.last_reported_time:type_name -> google.protobuf.Timestamp - 271, // 305: openshell.v1.SandboxProvisioning.configuration_change_time:type_name -> google.protobuf.Timestamp - 271, // 306: openshell.v1.SandboxProvisioning.first_rejection_time:type_name -> google.protobuf.Timestamp - 271, // 307: openshell.v1.SandboxProvisioning.deadline:type_name -> google.protobuf.Timestamp - 271, // 308: openshell.v1.SandboxProvisioning.timeout_time:type_name -> google.protobuf.Timestamp - 271, // 309: openshell.v1.SandboxProvisioning.cleanup_completed_time:type_name -> google.protobuf.Timestamp - 271, // 310: openshell.v1.SandboxProvisioning.cleanup_retry_time:type_name -> google.protobuf.Timestamp - 271, // 311: openshell.v1.SandboxProvisioning.attachment_change_time:type_name -> google.protobuf.Timestamp - 271, // 312: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp - 271, // 313: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp - 125, // 314: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry.value:type_name -> openshell.v1.ProviderProfileCredential - 153, // 315: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry.value:type_name -> openshell.v1.StaticCredentialBinding - 22, // 316: openshell.v1.OpenShell.Health:input_type -> openshell.v1.HealthRequest - 24, // 317: openshell.v1.OpenShell.GetCurrentUser:input_type -> openshell.v1.GetCurrentUserRequest - 26, // 318: openshell.v1.OpenShell.GetGatewayInfo:input_type -> openshell.v1.GetGatewayInfoRequest - 50, // 319: openshell.v1.OpenShell.CreateSandbox:input_type -> openshell.v1.CreateSandboxRequest - 58, // 320: openshell.v1.OpenShell.BeginRootfsTarStaging:input_type -> openshell.v1.BeginRootfsTarStagingRequest - 60, // 321: openshell.v1.OpenShell.GetSandbox:input_type -> openshell.v1.GetSandboxRequest - 61, // 322: openshell.v1.OpenShell.ListSandboxes:input_type -> openshell.v1.ListSandboxesRequest - 51, // 323: openshell.v1.OpenShell.CreateSandboxTemplate:input_type -> openshell.v1.CreateSandboxTemplateRequest - 52, // 324: openshell.v1.OpenShell.GetSandboxTemplate:input_type -> openshell.v1.GetSandboxTemplateRequest - 53, // 325: openshell.v1.OpenShell.ListSandboxTemplates:input_type -> openshell.v1.ListSandboxTemplatesRequest - 54, // 326: openshell.v1.OpenShell.DeleteSandboxTemplate:input_type -> openshell.v1.DeleteSandboxTemplateRequest - 62, // 327: openshell.v1.OpenShell.ListSandboxProviders:input_type -> openshell.v1.ListSandboxProvidersRequest - 63, // 328: openshell.v1.OpenShell.AttachSandboxProvider:input_type -> openshell.v1.AttachSandboxProviderRequest - 64, // 329: openshell.v1.OpenShell.DetachSandboxProvider:input_type -> openshell.v1.DetachSandboxProviderRequest - 80, // 330: openshell.v1.OpenShell.GetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest - 65, // 331: openshell.v1.OpenShell.DeleteSandbox:input_type -> openshell.v1.DeleteSandboxRequest - 66, // 332: openshell.v1.OpenShell.StopSandbox:input_type -> openshell.v1.StopSandboxRequest - 67, // 333: openshell.v1.OpenShell.StartSandbox:input_type -> openshell.v1.StartSandboxRequest - 85, // 334: openshell.v1.OpenShell.CreateSshSession:input_type -> openshell.v1.CreateSshSessionRequest - 87, // 335: openshell.v1.OpenShell.ExposeService:input_type -> openshell.v1.ExposeServiceRequest - 88, // 336: openshell.v1.OpenShell.GetService:input_type -> openshell.v1.GetServiceRequest - 89, // 337: openshell.v1.OpenShell.ListServices:input_type -> openshell.v1.ListServicesRequest - 91, // 338: openshell.v1.OpenShell.DeleteService:input_type -> openshell.v1.DeleteServiceRequest - 95, // 339: openshell.v1.OpenShell.RevokeSshSession:input_type -> openshell.v1.RevokeSshSessionRequest - 97, // 340: openshell.v1.OpenShell.ExecSandbox:input_type -> openshell.v1.ExecSandboxRequest - 103, // 341: openshell.v1.OpenShell.ForwardTcp:input_type -> openshell.v1.TcpForwardFrame - 104, // 342: openshell.v1.OpenShell.ExecSandboxInteractive:input_type -> openshell.v1.ExecSandboxInput - 111, // 343: openshell.v1.OpenShell.CreateProvider:input_type -> openshell.v1.CreateProviderRequest - 112, // 344: openshell.v1.OpenShell.GetProvider:input_type -> openshell.v1.GetProviderRequest - 113, // 345: openshell.v1.OpenShell.ListProviders:input_type -> openshell.v1.ListProvidersRequest - 118, // 346: openshell.v1.OpenShell.ListProviderProfiles:input_type -> openshell.v1.ListProviderProfilesRequest - 119, // 347: openshell.v1.OpenShell.GetProviderProfile:input_type -> openshell.v1.GetProviderProfileRequest - 142, // 348: openshell.v1.OpenShell.ImportProviderProfiles:input_type -> openshell.v1.ImportProviderProfilesRequest - 144, // 349: openshell.v1.OpenShell.UpdateProviderProfiles:input_type -> openshell.v1.UpdateProviderProfilesRequest - 146, // 350: openshell.v1.OpenShell.LintProviderProfiles:input_type -> openshell.v1.LintProviderProfilesRequest - 114, // 351: openshell.v1.OpenShell.UpdateProvider:input_type -> openshell.v1.UpdateProviderRequest - 131, // 352: openshell.v1.OpenShell.GetProviderRefreshStatus:input_type -> openshell.v1.GetProviderRefreshStatusRequest - 133, // 353: openshell.v1.OpenShell.ConfigureProviderRefresh:input_type -> openshell.v1.ConfigureProviderRefreshRequest - 135, // 354: openshell.v1.OpenShell.RotateProviderCredential:input_type -> openshell.v1.RotateProviderCredentialRequest - 137, // 355: openshell.v1.OpenShell.DeleteProviderRefresh:input_type -> openshell.v1.DeleteProviderRefreshRequest - 115, // 356: openshell.v1.OpenShell.DeleteProvider:input_type -> openshell.v1.DeleteProviderRequest - 149, // 357: openshell.v1.OpenShell.DeleteProviderProfile:input_type -> openshell.v1.DeleteProviderProfileRequest - 286, // 358: openshell.v1.OpenShell.GetSandboxConfig:input_type -> openshell.sandbox.v1.GetSandboxConfigRequest - 287, // 359: openshell.v1.OpenShell.GetGatewayConfig:input_type -> openshell.sandbox.v1.GetGatewayConfigRequest - 157, // 360: openshell.v1.OpenShell.UpdateConfig:input_type -> openshell.v1.UpdateConfigRequest - 167, // 361: openshell.v1.OpenShell.GetSandboxPolicyStatus:input_type -> openshell.v1.GetSandboxPolicyStatusRequest - 169, // 362: openshell.v1.OpenShell.ListSandboxPolicies:input_type -> openshell.v1.ListSandboxPoliciesRequest - 171, // 363: openshell.v1.OpenShell.ReportPolicyStatus:input_type -> openshell.v1.ReportPolicyStatusRequest - 244, // 364: openshell.v1.OpenShell.ReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest - 82, // 365: openshell.v1.OpenShell.ReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest - 174, // 366: openshell.v1.OpenShell.ReportSandboxConfiguration:input_type -> openshell.v1.ReportSandboxConfigurationRequest - 151, // 367: openshell.v1.OpenShell.GetSandboxProviderEnvironment:input_type -> openshell.v1.GetSandboxProviderEnvironmentRequest - 155, // 368: openshell.v1.OpenShell.ExchangeProviderSubjectToken:input_type -> openshell.v1.ExchangeProviderSubjectTokenRequest - 177, // 369: openshell.v1.OpenShell.GetSandboxLogs:input_type -> openshell.v1.GetSandboxLogsRequest - 178, // 370: openshell.v1.OpenShell.PushSandboxLogs:input_type -> openshell.v1.PushSandboxLogsRequest - 181, // 371: openshell.v1.OpenShell.ConnectSupervisor:input_type -> openshell.v1.SupervisorMessage - 188, // 372: openshell.v1.OpenShell.ReportMainProcessExit:input_type -> openshell.v1.ReportMainProcessExitRequest - 190, // 373: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest - 196, // 374: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame - 198, // 375: openshell.v1.OpenShell.PeerRelay:input_type -> openshell.v1.PeerRelayFrame - 82, // 376: openshell.v1.OpenShell.PeerReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest - 244, // 377: openshell.v1.OpenShell.PeerReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest - 80, // 378: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest - 107, // 379: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest - 207, // 380: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest - 209, // 381: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest - 211, // 382: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest - 213, // 383: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest - 216, // 384: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest - 218, // 385: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest - 220, // 386: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest - 222, // 387: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest - 224, // 388: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest - 18, // 389: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest - 20, // 390: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest - 227, // 391: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest - 229, // 392: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest - 231, // 393: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest - 233, // 394: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest - 236, // 395: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest - 238, // 396: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest - 240, // 397: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest - 23, // 398: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse - 25, // 399: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse - 27, // 400: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse - 68, // 401: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse - 59, // 402: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse - 68, // 403: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse - 69, // 404: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse - 55, // 405: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 55, // 406: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 56, // 407: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse - 57, // 408: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse - 70, // 409: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse - 71, // 410: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse - 72, // 411: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse - 81, // 412: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse - 84, // 413: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse - 68, // 414: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse - 68, // 415: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse - 86, // 416: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse - 94, // 417: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse - 94, // 418: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse - 90, // 419: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse - 92, // 420: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse - 96, // 421: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse - 101, // 422: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent - 103, // 423: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame - 101, // 424: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent - 116, // 425: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse - 116, // 426: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse - 117, // 427: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse - 141, // 428: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse - 140, // 429: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse - 143, // 430: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse - 145, // 431: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse - 147, // 432: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse - 116, // 433: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse - 132, // 434: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse - 134, // 435: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse - 136, // 436: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse - 138, // 437: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse - 148, // 438: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse - 150, // 439: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse - 288, // 440: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse - 289, // 441: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse - 166, // 442: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse - 168, // 443: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse - 170, // 444: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse - 172, // 445: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse - 245, // 446: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse - 83, // 447: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse - 175, // 448: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse - 154, // 449: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse - 156, // 450: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse - 180, // 451: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse - 179, // 452: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse - 182, // 453: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage - 189, // 454: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse - 191, // 455: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse - 196, // 456: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame - 198, // 457: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame - 83, // 458: openshell.v1.OpenShell.PeerReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse - 245, // 459: openshell.v1.OpenShell.PeerReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse - 81, // 460: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse - 108, // 461: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent - 208, // 462: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse - 210, // 463: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse - 212, // 464: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse - 214, // 465: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse - 217, // 466: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse - 219, // 467: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse - 221, // 468: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse - 223, // 469: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse - 226, // 470: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse - 19, // 471: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse - 21, // 472: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse - 228, // 473: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse - 230, // 474: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse - 232, // 475: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse - 234, // 476: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse - 237, // 477: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse - 239, // 478: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse - 241, // 479: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse - 398, // [398:480] is the sub-list for method output_type - 316, // [316:398] is the sub-list for method input_type - 316, // [316:316] is the sub-list for extension type_name - 316, // [316:316] is the sub-list for extension extendee - 0, // [0:316] is the sub-list for field type_name + 31, // 9: openshell.v1.ComputeDriverInfo.capabilities:type_name -> openshell.v1.ComputeDriverCapabilities + 32, // 10: openshell.v1.ComputeDriverCapabilities.resource_capabilities:type_name -> openshell.v1.ResourceCapabilities + 33, // 11: openshell.v1.ResourceCapabilities.cpu:type_name -> openshell.v1.CpuResourceCapabilities + 34, // 12: openshell.v1.ResourceCapabilities.memory:type_name -> openshell.v1.MemoryResourceCapabilities + 35, // 13: openshell.v1.ResourceCapabilities.gpu:type_name -> openshell.v1.GpuResourceCapabilities + 273, // 14: openshell.v1.Sandbox.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 37, // 15: openshell.v1.Sandbox.spec:type_name -> openshell.v1.SandboxSpec + 48, // 16: openshell.v1.Sandbox.status:type_name -> openshell.v1.SandboxStatus + 47, // 17: openshell.v1.Sandbox.created_from_workload_template:type_name -> openshell.v1.SandboxWorkloadTemplateProvenance + 250, // 18: openshell.v1.SandboxSpec.environment:type_name -> openshell.v1.SandboxSpec.EnvironmentEntry + 40, // 19: openshell.v1.SandboxSpec.template:type_name -> openshell.v1.SandboxTemplate + 274, // 20: openshell.v1.SandboxSpec.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 38, // 21: openshell.v1.SandboxSpec.resource_requirements:type_name -> openshell.v1.ResourceRequirements + 16, // 22: openshell.v1.SandboxSpec.restart_policy:type_name -> openshell.v1.SandboxRestartPolicy + 39, // 23: openshell.v1.ResourceRequirements.gpu:type_name -> openshell.v1.GpuResourceRequirements + 251, // 24: openshell.v1.SandboxTemplate.labels:type_name -> openshell.v1.SandboxTemplate.LabelsEntry + 252, // 25: openshell.v1.SandboxTemplate.annotations:type_name -> openshell.v1.SandboxTemplate.AnnotationsEntry + 253, // 26: openshell.v1.SandboxTemplate.environment:type_name -> openshell.v1.SandboxTemplate.EnvironmentEntry + 275, // 27: openshell.v1.SandboxTemplate.resources:type_name -> google.protobuf.Struct + 275, // 28: openshell.v1.SandboxTemplate.driver_config:type_name -> google.protobuf.Struct + 273, // 29: openshell.v1.SandboxWorkloadTemplate.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 42, // 30: openshell.v1.SandboxWorkloadTemplate.spec:type_name -> openshell.v1.SandboxWorkloadTemplateSpec + 43, // 31: openshell.v1.SandboxWorkloadTemplateSpec.workload:type_name -> openshell.v1.SandboxWorkloadConfig + 275, // 32: openshell.v1.SandboxWorkloadTemplateSpec.driver_config:type_name -> google.protobuf.Struct + 45, // 33: openshell.v1.SandboxWorkloadTemplateSpec.desired_service_level:type_name -> openshell.v1.SandboxServiceLevel + 254, // 34: openshell.v1.SandboxWorkloadConfig.environment:type_name -> openshell.v1.SandboxWorkloadConfig.EnvironmentEntry + 44, // 35: openshell.v1.SandboxWorkloadConfig.resources:type_name -> openshell.v1.SandboxResources + 39, // 36: openshell.v1.SandboxResources.gpu:type_name -> openshell.v1.GpuResourceRequirements + 46, // 37: openshell.v1.SandboxServiceLevel.startup:type_name -> openshell.v1.SandboxStartup + 276, // 38: openshell.v1.SandboxStartup.ready_within:type_name -> google.protobuf.Duration + 49, // 39: openshell.v1.SandboxStatus.conditions:type_name -> openshell.v1.SandboxCondition + 1, // 40: openshell.v1.SandboxStatus.phase:type_name -> openshell.v1.SandboxPhase + 247, // 41: openshell.v1.SandboxStatus.endpoint_statuses:type_name -> openshell.v1.EndpointStatus + 174, // 42: openshell.v1.SandboxStatus.configuration_admission:type_name -> openshell.v1.SandboxConfigurationAdmission + 248, // 43: openshell.v1.SandboxStatus.provisioning:type_name -> openshell.v1.SandboxProvisioning + 272, // 44: openshell.v1.SandboxStatus.next_restart_time:type_name -> google.protobuf.Timestamp + 272, // 45: openshell.v1.SandboxStatus.main_process_started_time:type_name -> google.protobuf.Timestamp + 272, // 46: openshell.v1.SandboxCondition.transition_time:type_name -> google.protobuf.Timestamp + 272, // 47: openshell.v1.PlatformEvent.event_time:type_name -> google.protobuf.Timestamp + 255, // 48: openshell.v1.PlatformEvent.metadata:type_name -> openshell.v1.PlatformEvent.MetadataEntry + 277, // 49: openshell.v1.CreateSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 37, // 50: openshell.v1.CreateSandboxRequest.spec:type_name -> openshell.v1.SandboxSpec + 256, // 51: openshell.v1.CreateSandboxRequest.labels:type_name -> openshell.v1.CreateSandboxRequest.LabelsEntry + 257, // 52: openshell.v1.CreateSandboxRequest.annotations:type_name -> openshell.v1.CreateSandboxRequest.AnnotationsEntry + 249, // 53: openshell.v1.CreateSandboxRequest.service_exposures:type_name -> openshell.v1.SandboxServiceExposure + 277, // 54: openshell.v1.CreateSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 41, // 55: openshell.v1.CreateSandboxTemplateRequest.template:type_name -> openshell.v1.SandboxWorkloadTemplate + 277, // 56: openshell.v1.GetSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 57: openshell.v1.ListSandboxTemplatesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 58: openshell.v1.DeleteSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 41, // 59: openshell.v1.SandboxTemplateResponse.template:type_name -> openshell.v1.SandboxWorkloadTemplate + 41, // 60: openshell.v1.ListSandboxTemplatesResponse.templates:type_name -> openshell.v1.SandboxWorkloadTemplate + 17, // 61: openshell.v1.DeleteSandboxTemplateResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 277, // 62: openshell.v1.BeginRootfsTarStagingRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 272, // 63: openshell.v1.BeginRootfsTarStagingResponse.expiration_time:type_name -> google.protobuf.Timestamp + 277, // 64: openshell.v1.GetSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 65: openshell.v1.ListSandboxesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 66: openshell.v1.ListSandboxProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 67: openshell.v1.AttachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 68: openshell.v1.DetachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 69: openshell.v1.DeleteSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 70: openshell.v1.StopSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 71: openshell.v1.StartSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 36, // 72: openshell.v1.SandboxResponse.sandbox:type_name -> openshell.v1.Sandbox + 258, // 73: openshell.v1.SandboxResponse.service_urls:type_name -> openshell.v1.SandboxResponse.ServiceUrlsEntry + 36, // 74: openshell.v1.ListSandboxesResponse.sandboxes:type_name -> openshell.v1.Sandbox + 278, // 75: openshell.v1.ListSandboxProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider + 36, // 76: openshell.v1.AttachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox + 78, // 77: openshell.v1.AttachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt + 36, // 78: openshell.v1.DetachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox + 78, // 79: openshell.v1.DetachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt + 76, // 80: openshell.v1.ConfigSnapshotRevision.sandbox_config:type_name -> openshell.v1.SandboxConfigRevision + 74, // 81: openshell.v1.ConfigSnapshotRevision.provider_target:type_name -> openshell.v1.ProviderDesiredIdentity + 279, // 82: openshell.v1.SandboxConfigRevision.policy_source:type_name -> openshell.sandbox.v1.PolicySource + 5, // 83: openshell.v1.ConfigUpdateOperation.component:type_name -> openshell.v1.ConfigComponent + 75, // 84: openshell.v1.ConfigUpdateOperation.target_revision:type_name -> openshell.v1.ConfigSnapshotRevision + 7, // 85: openshell.v1.ConfigUpdateOperation.state:type_name -> openshell.v1.ConfigUpdateOperationState + 6, // 86: openshell.v1.ConfigUpdateOperation.outcome:type_name -> openshell.v1.ConfigApplyOutcome + 272, // 87: openshell.v1.ConfigUpdateOperation.created_time:type_name -> google.protobuf.Timestamp + 272, // 88: openshell.v1.ConfigUpdateOperation.updated_time:type_name -> google.protobuf.Timestamp + 272, // 89: openshell.v1.ConfigUpdateOperation.completed_time:type_name -> google.protobuf.Timestamp + 2, // 90: openshell.v1.ProviderMutationReceipt.kind:type_name -> openshell.v1.ProviderMutationKind + 74, // 91: openshell.v1.ProviderMutationReceipt.desired:type_name -> openshell.v1.ProviderDesiredIdentity + 272, // 92: openshell.v1.ProviderMutationReceipt.persisted_time:type_name -> google.protobuf.Timestamp + 4, // 93: openshell.v1.ProviderReadinessObservation.reason:type_name -> openshell.v1.ProviderReadinessReason + 78, // 94: openshell.v1.ProviderReadinessStatus.receipt:type_name -> openshell.v1.ProviderMutationReceipt + 3, // 95: openshell.v1.ProviderReadinessStatus.state:type_name -> openshell.v1.ProviderReadinessState + 4, // 96: openshell.v1.ProviderReadinessStatus.reason:type_name -> openshell.v1.ProviderReadinessReason + 79, // 97: openshell.v1.ProviderReadinessStatus.observed:type_name -> openshell.v1.ProviderReadinessObservation + 272, // 98: openshell.v1.ProviderReadinessStatus.observed_time:type_name -> google.protobuf.Timestamp + 272, // 99: openshell.v1.ProviderReadinessStatus.evaluated_time:type_name -> google.protobuf.Timestamp + 77, // 100: openshell.v1.ProviderReadinessStatus.operation:type_name -> openshell.v1.ConfigUpdateOperation + 277, // 101: openshell.v1.GetSandboxProviderStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 80, // 102: openshell.v1.GetSandboxProviderStatusResponse.status:type_name -> openshell.v1.ProviderReadinessStatus + 79, // 103: openshell.v1.ReportProviderReadinessRequest.observation:type_name -> openshell.v1.ProviderReadinessObservation + 276, // 104: openshell.v1.ReportProviderReadinessResponse.report_interval:type_name -> google.protobuf.Duration + 276, // 105: openshell.v1.ReportProviderReadinessResponse.observation_ttl:type_name -> google.protobuf.Duration + 17, // 106: openshell.v1.DeleteSandboxResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 277, // 107: openshell.v1.CreateSshSessionRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 272, // 108: openshell.v1.CreateSshSessionResponse.expiration_time:type_name -> google.protobuf.Timestamp + 277, // 109: openshell.v1.ExposeServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 110: openshell.v1.GetServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 111: openshell.v1.ListServicesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 95, // 112: openshell.v1.ListServicesResponse.services:type_name -> openshell.v1.ServiceEndpointResponse + 277, // 113: openshell.v1.DeleteServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 17, // 114: openshell.v1.DeleteServiceResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 273, // 115: openshell.v1.ServiceEndpoint.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 94, // 116: openshell.v1.ServiceEndpointResponse.endpoint:type_name -> openshell.v1.ServiceEndpoint + 17, // 117: openshell.v1.RevokeSshSessionResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 277, // 118: openshell.v1.ExecSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 259, // 119: openshell.v1.ExecSandboxRequest.environment:type_name -> openshell.v1.ExecSandboxRequest.EnvironmentEntry + 276, // 120: openshell.v1.ExecSandboxRequest.execution_timeout:type_name -> google.protobuf.Duration + 99, // 121: openshell.v1.ExecSandboxEvent.stdout:type_name -> openshell.v1.ExecSandboxStdout + 100, // 122: openshell.v1.ExecSandboxEvent.stderr:type_name -> openshell.v1.ExecSandboxStderr + 101, // 123: openshell.v1.ExecSandboxEvent.exit:type_name -> openshell.v1.ExecSandboxExit + 194, // 124: openshell.v1.TcpForwardInit.ssh:type_name -> openshell.v1.SshRelayTarget + 195, // 125: openshell.v1.TcpForwardInit.tcp:type_name -> openshell.v1.TcpRelayTarget + 103, // 126: openshell.v1.TcpForwardFrame.init:type_name -> openshell.v1.TcpForwardInit + 98, // 127: openshell.v1.ExecSandboxInput.start:type_name -> openshell.v1.ExecSandboxRequest + 106, // 128: openshell.v1.ExecSandboxInput.resize:type_name -> openshell.v1.ExecSandboxWindowResize + 273, // 129: openshell.v1.SshSession.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 272, // 130: openshell.v1.SshSession.expiration_time:type_name -> google.protobuf.Timestamp + 277, // 131: openshell.v1.WatchSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 272, // 132: openshell.v1.WatchSandboxRequest.since_time:type_name -> google.protobuf.Timestamp + 36, // 133: openshell.v1.SandboxStreamEvent.sandbox:type_name -> openshell.v1.Sandbox + 110, // 134: openshell.v1.SandboxStreamEvent.log:type_name -> openshell.v1.SandboxLogLine + 50, // 135: openshell.v1.SandboxStreamEvent.event:type_name -> openshell.v1.PlatformEvent + 111, // 136: openshell.v1.SandboxStreamEvent.warning:type_name -> openshell.v1.SandboxStreamWarning + 207, // 137: openshell.v1.SandboxStreamEvent.draft_policy_update:type_name -> openshell.v1.DraftPolicyUpdate + 272, // 138: openshell.v1.SandboxLogLine.event_time:type_name -> google.protobuf.Timestamp + 260, // 139: openshell.v1.SandboxLogLine.fields:type_name -> openshell.v1.SandboxLogLine.FieldsEntry + 277, // 140: openshell.v1.CreateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 278, // 141: openshell.v1.CreateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider + 277, // 142: openshell.v1.GetProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 143: openshell.v1.ListProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 144: openshell.v1.UpdateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 278, // 145: openshell.v1.UpdateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider + 261, // 146: openshell.v1.UpdateProviderRequest.credential_expiration_times:type_name -> openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry + 277, // 147: openshell.v1.DeleteProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 278, // 148: openshell.v1.ProviderResponse.provider:type_name -> openshell.datamodel.v1.Provider + 78, // 149: openshell.v1.ProviderResponse.target_receipts:type_name -> openshell.v1.ProviderMutationReceipt + 278, // 150: openshell.v1.ListProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider + 277, // 151: openshell.v1.ListProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 152: openshell.v1.GetProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 140, // 153: openshell.v1.ProviderProfileImportItem.profile:type_name -> openshell.v1.ProviderProfile + 276, // 154: openshell.v1.ProviderCredentialTokenGrant.cache_ttl:type_name -> google.protobuf.Duration + 123, // 155: openshell.v1.ProviderCredentialTokenGrant.audience_overrides:type_name -> openshell.v1.ProviderCredentialTokenGrantAudienceOverride + 8, // 156: openshell.v1.ProviderCredentialTokenGrant.grant_type:type_name -> openshell.v1.ProviderCredentialTokenGrantType + 124, // 157: openshell.v1.ProviderCredentialTokenGrant.subject_token:type_name -> openshell.v1.ProviderCredentialTokenGrantSubjectToken + 129, // 158: openshell.v1.ProviderProfileCredential.refresh:type_name -> openshell.v1.ProviderCredentialRefresh + 125, // 159: openshell.v1.ProviderProfileCredential.token_grant:type_name -> openshell.v1.ProviderCredentialTokenGrant + 9, // 160: openshell.v1.ProviderCredentialRefresh.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy + 276, // 161: openshell.v1.ProviderCredentialRefresh.refresh_before:type_name -> google.protobuf.Duration + 276, // 162: openshell.v1.ProviderCredentialRefresh.max_lifetime:type_name -> google.protobuf.Duration + 127, // 163: openshell.v1.ProviderCredentialRefresh.material:type_name -> openshell.v1.ProviderCredentialRefreshMaterial + 128, // 164: openshell.v1.ProviderCredentialRefresh.additional_outputs:type_name -> openshell.v1.ProviderCredentialRefreshOutput + 9, // 165: openshell.v1.ProviderCredentialRefreshStatus.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy + 272, // 166: openshell.v1.ProviderCredentialRefreshStatus.expiration_time:type_name -> google.protobuf.Timestamp + 272, // 167: openshell.v1.ProviderCredentialRefreshStatus.next_refresh_time:type_name -> google.protobuf.Timestamp + 272, // 168: openshell.v1.ProviderCredentialRefreshStatus.last_refresh_time:type_name -> google.protobuf.Timestamp + 15, // 169: openshell.v1.ProviderCredentialRefreshStatus.recovery_action:type_name -> openshell.v1.ProviderCredentialRefreshRecoveryAction + 272, // 170: openshell.v1.ProviderCredentialRefreshStatus.last_error_time:type_name -> google.protobuf.Timestamp + 277, // 171: openshell.v1.GetProviderRefreshStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 130, // 172: openshell.v1.GetProviderRefreshStatusResponse.credentials:type_name -> openshell.v1.ProviderCredentialRefreshStatus + 277, // 173: openshell.v1.ConfigureProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 9, // 174: openshell.v1.ConfigureProviderRefreshRequest.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy + 262, // 175: openshell.v1.ConfigureProviderRefreshRequest.material:type_name -> openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry + 272, // 176: openshell.v1.ConfigureProviderRefreshRequest.expiration_time:type_name -> google.protobuf.Timestamp + 130, // 177: openshell.v1.ConfigureProviderRefreshResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus + 277, // 178: openshell.v1.RotateProviderCredentialRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 130, // 179: openshell.v1.RotateProviderCredentialResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus + 277, // 180: openshell.v1.DeleteProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 17, // 181: openshell.v1.DeleteProviderRefreshResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 10, // 182: openshell.v1.ProviderProfile.category:type_name -> openshell.v1.ProviderProfileCategory + 126, // 183: openshell.v1.ProviderProfile.credentials:type_name -> openshell.v1.ProviderProfileCredential + 280, // 184: openshell.v1.ProviderProfile.endpoints:type_name -> openshell.sandbox.v1.NetworkEndpoint + 281, // 185: openshell.v1.ProviderProfile.binaries:type_name -> openshell.sandbox.v1.NetworkBinary + 131, // 186: openshell.v1.ProviderProfile.discovery:type_name -> openshell.v1.ProviderProfileDiscovery + 263, // 187: openshell.v1.ProviderProfile.annotations:type_name -> openshell.v1.ProviderProfile.AnnotationsEntry + 140, // 188: openshell.v1.ProviderProfileResponse.profile:type_name -> openshell.v1.ProviderProfile + 140, // 189: openshell.v1.ListProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile + 277, // 190: openshell.v1.ImportProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 121, // 191: openshell.v1.ImportProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem + 122, // 192: openshell.v1.ImportProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic + 140, // 193: openshell.v1.ImportProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile + 277, // 194: openshell.v1.UpdateProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 121, // 195: openshell.v1.UpdateProviderProfilesRequest.profile:type_name -> openshell.v1.ProviderProfileImportItem + 122, // 196: openshell.v1.UpdateProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic + 140, // 197: openshell.v1.UpdateProviderProfilesResponse.profile:type_name -> openshell.v1.ProviderProfile + 277, // 198: openshell.v1.LintProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 121, // 199: openshell.v1.LintProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem + 122, // 200: openshell.v1.LintProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic + 17, // 201: openshell.v1.DeleteProviderResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 277, // 202: openshell.v1.DeleteProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 17, // 203: openshell.v1.DeleteProviderProfileResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 153, // 204: openshell.v1.StaticCredentialBinding.endpoints:type_name -> openshell.v1.StaticCredentialEndpointBinding + 264, // 205: openshell.v1.GetSandboxProviderEnvironmentResponse.environment:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry + 265, // 206: openshell.v1.GetSandboxProviderEnvironmentResponse.credential_expiration_times:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry + 266, // 207: openshell.v1.GetSandboxProviderEnvironmentResponse.dynamic_credentials:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry + 267, // 208: openshell.v1.GetSandboxProviderEnvironmentResponse.static_credential_bindings:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry + 4, // 209: openshell.v1.GetSandboxProviderEnvironmentResponse.readiness_reason:type_name -> openshell.v1.ProviderReadinessReason + 276, // 210: openshell.v1.ExchangeProviderSubjectTokenResponse.expires_after:type_name -> google.protobuf.Duration + 277, // 211: openshell.v1.UpdateConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 212: openshell.v1.UpdateConfigRequest.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 282, // 213: openshell.v1.UpdateConfigRequest.setting_value:type_name -> openshell.sandbox.v1.SettingValue + 159, // 214: openshell.v1.UpdateConfigRequest.merge_operations:type_name -> openshell.v1.PolicyMergeOperation + 268, // 215: openshell.v1.UpdateConfigRequest.annotations:type_name -> openshell.v1.UpdateConfigRequest.AnnotationsEntry + 160, // 216: openshell.v1.PolicyMergeOperation.add_rule:type_name -> openshell.v1.AddNetworkRule + 161, // 217: openshell.v1.PolicyMergeOperation.remove_endpoint:type_name -> openshell.v1.RemoveNetworkEndpoint + 162, // 218: openshell.v1.PolicyMergeOperation.remove_rule:type_name -> openshell.v1.RemoveNetworkRule + 164, // 219: openshell.v1.PolicyMergeOperation.add_deny_rules:type_name -> openshell.v1.AddDenyRules + 165, // 220: openshell.v1.PolicyMergeOperation.add_allow_rules:type_name -> openshell.v1.AddAllowRules + 166, // 221: openshell.v1.PolicyMergeOperation.remove_binary:type_name -> openshell.v1.RemoveNetworkBinary + 283, // 222: openshell.v1.AddNetworkRule.rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 281, // 223: openshell.v1.L7RuleTarget.binaries:type_name -> openshell.sandbox.v1.NetworkBinary + 284, // 224: openshell.v1.AddDenyRules.deny_rules:type_name -> openshell.sandbox.v1.L7DenyRule + 163, // 225: openshell.v1.AddDenyRules.target:type_name -> openshell.v1.L7RuleTarget + 285, // 226: openshell.v1.AddAllowRules.rules:type_name -> openshell.sandbox.v1.L7Rule + 163, // 227: openshell.v1.AddAllowRules.target:type_name -> openshell.v1.L7RuleTarget + 269, // 228: openshell.v1.UpdateConfigResponse.annotations:type_name -> openshell.v1.UpdateConfigResponse.AnnotationsEntry + 277, // 229: openshell.v1.GetSandboxPolicyStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 177, // 230: openshell.v1.GetSandboxPolicyStatusResponse.revision:type_name -> openshell.v1.SandboxPolicyRevision + 277, // 231: openshell.v1.ListSandboxPoliciesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 177, // 232: openshell.v1.ListSandboxPoliciesResponse.revisions:type_name -> openshell.v1.SandboxPolicyRevision + 12, // 233: openshell.v1.ReportPolicyStatusRequest.status:type_name -> openshell.v1.PolicyStatus + 11, // 234: openshell.v1.SandboxConfigurationAdmission.state:type_name -> openshell.v1.ConfigurationAdmissionState + 174, // 235: openshell.v1.ReportSandboxConfigurationRequest.admission:type_name -> openshell.v1.SandboxConfigurationAdmission + 12, // 236: openshell.v1.SandboxPolicyRevision.status:type_name -> openshell.v1.PolicyStatus + 272, // 237: openshell.v1.SandboxPolicyRevision.created_time:type_name -> google.protobuf.Timestamp + 272, // 238: openshell.v1.SandboxPolicyRevision.loaded_time:type_name -> google.protobuf.Timestamp + 274, // 239: openshell.v1.SandboxPolicyRevision.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 270, // 240: openshell.v1.SandboxPolicyRevision.provenance:type_name -> openshell.v1.SandboxPolicyRevision.ProvenanceEntry + 277, // 241: openshell.v1.GetSandboxLogsRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 272, // 242: openshell.v1.GetSandboxLogsRequest.since_time:type_name -> google.protobuf.Timestamp + 110, // 243: openshell.v1.PushSandboxLogsRequest.logs:type_name -> openshell.v1.SandboxLogLine + 110, // 244: openshell.v1.GetSandboxLogsResponse.logs:type_name -> openshell.v1.SandboxLogLine + 184, // 245: openshell.v1.SupervisorMessage.hello:type_name -> openshell.v1.SupervisorHello + 187, // 246: openshell.v1.SupervisorMessage.heartbeat:type_name -> openshell.v1.SupervisorHeartbeat + 200, // 247: openshell.v1.SupervisorMessage.relay_open_result:type_name -> openshell.v1.RelayOpenResult + 201, // 248: openshell.v1.SupervisorMessage.relay_close:type_name -> openshell.v1.RelayClose + 185, // 249: openshell.v1.GatewayMessage.session_accepted:type_name -> openshell.v1.SessionAccepted + 186, // 250: openshell.v1.GatewayMessage.session_rejected:type_name -> openshell.v1.SessionRejected + 188, // 251: openshell.v1.GatewayMessage.heartbeat:type_name -> openshell.v1.GatewayHeartbeat + 193, // 252: openshell.v1.GatewayMessage.relay_open:type_name -> openshell.v1.RelayOpen + 201, // 253: openshell.v1.GatewayMessage.relay_close:type_name -> openshell.v1.RelayClose + 276, // 254: openshell.v1.SessionAccepted.heartbeat_interval:type_name -> google.protobuf.Duration + 194, // 255: openshell.v1.RelayOpen.ssh:type_name -> openshell.v1.SshRelayTarget + 195, // 256: openshell.v1.RelayOpen.tcp:type_name -> openshell.v1.TcpRelayTarget + 196, // 257: openshell.v1.RelayFrame.init:type_name -> openshell.v1.RelayInit + 193, // 258: openshell.v1.PeerRelayInit.relay_open:type_name -> openshell.v1.RelayOpen + 198, // 259: openshell.v1.PeerRelayFrame.init:type_name -> openshell.v1.PeerRelayInit + 272, // 260: openshell.v1.DenialSummary.first_seen_time:type_name -> google.protobuf.Timestamp + 272, // 261: openshell.v1.DenialSummary.last_seen_time:type_name -> google.protobuf.Timestamp + 202, // 262: openshell.v1.DenialSummary.l7_request_samples:type_name -> openshell.v1.L7RequestSample + 204, // 263: openshell.v1.NetworkActivitySummary.denials_by_group:type_name -> openshell.v1.DenialGroupCount + 283, // 264: openshell.v1.PolicyChunk.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 272, // 265: openshell.v1.PolicyChunk.created_time:type_name -> google.protobuf.Timestamp + 272, // 266: openshell.v1.PolicyChunk.decided_time:type_name -> google.protobuf.Timestamp + 272, // 267: openshell.v1.PolicyChunk.first_seen_time:type_name -> google.protobuf.Timestamp + 272, // 268: openshell.v1.PolicyChunk.last_seen_time:type_name -> google.protobuf.Timestamp + 274, // 269: openshell.v1.PolicyChunk.current_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 274, // 270: openshell.v1.PolicyChunk.candidate_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 277, // 271: openshell.v1.SubmitPolicyAnalysisRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 203, // 272: openshell.v1.SubmitPolicyAnalysisRequest.summaries:type_name -> openshell.v1.DenialSummary + 206, // 273: openshell.v1.SubmitPolicyAnalysisRequest.proposed_chunks:type_name -> openshell.v1.PolicyChunk + 205, // 274: openshell.v1.SubmitPolicyAnalysisRequest.network_activity_summaries:type_name -> openshell.v1.NetworkActivitySummary + 277, // 275: openshell.v1.GetDraftPolicyRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 206, // 276: openshell.v1.GetDraftPolicyResponse.chunks:type_name -> openshell.v1.PolicyChunk + 272, // 277: openshell.v1.GetDraftPolicyResponse.last_analyzed_time:type_name -> google.protobuf.Timestamp + 277, // 278: openshell.v1.ApproveDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 279: openshell.v1.RejectDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 280: openshell.v1.ApproveAllDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 216, // 281: openshell.v1.ApproveAllDraftChunksRequest.approvals:type_name -> openshell.v1.DraftChunkApproval + 277, // 282: openshell.v1.EditDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 283, // 283: openshell.v1.EditDraftChunkRequest.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 277, // 284: openshell.v1.UndoDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 285: openshell.v1.ClearDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 277, // 286: openshell.v1.GetDraftHistoryRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 272, // 287: openshell.v1.DraftHistoryEntry.event_time:type_name -> google.protobuf.Timestamp + 226, // 288: openshell.v1.GetDraftHistoryResponse.entries:type_name -> openshell.v1.DraftHistoryEntry + 271, // 289: openshell.v1.CreateWorkspaceRequest.labels:type_name -> openshell.v1.CreateWorkspaceRequest.LabelsEntry + 286, // 290: openshell.v1.CreateWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace + 286, // 291: openshell.v1.GetWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace + 286, // 292: openshell.v1.ListWorkspacesResponse.workspaces:type_name -> openshell.datamodel.v1.Workspace + 17, // 293: openshell.v1.DeleteWorkspaceResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 273, // 294: openshell.v1.WorkspaceMember.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 14, // 295: openshell.v1.WorkspaceMember.role:type_name -> openshell.v1.WorkspaceRole + 277, // 296: openshell.v1.AddWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 14, // 297: openshell.v1.AddWorkspaceMemberRequest.role:type_name -> openshell.v1.WorkspaceRole + 236, // 298: openshell.v1.AddWorkspaceMemberResponse.member:type_name -> openshell.v1.WorkspaceMember + 277, // 299: openshell.v1.RemoveWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 17, // 300: openshell.v1.RemoveWorkspaceMemberResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 277, // 301: openshell.v1.ListWorkspaceMembersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 236, // 302: openshell.v1.ListWorkspaceMembersResponse.members:type_name -> openshell.v1.WorkspaceMember + 272, // 303: openshell.v1.ExtensionServiceCredential.expiration_time:type_name -> google.protobuf.Timestamp + 18, // 304: openshell.v1.EndpointObservation.result:type_name -> openshell.v1.EndpointResult + 244, // 305: openshell.v1.ReportEndpointStatusRequest.observations:type_name -> openshell.v1.EndpointObservation + 18, // 306: openshell.v1.EndpointStatus.last_result:type_name -> openshell.v1.EndpointResult + 272, // 307: openshell.v1.EndpointStatus.last_reported_time:type_name -> google.protobuf.Timestamp + 272, // 308: openshell.v1.SandboxProvisioning.configuration_change_time:type_name -> google.protobuf.Timestamp + 272, // 309: openshell.v1.SandboxProvisioning.first_rejection_time:type_name -> google.protobuf.Timestamp + 272, // 310: openshell.v1.SandboxProvisioning.deadline:type_name -> google.protobuf.Timestamp + 272, // 311: openshell.v1.SandboxProvisioning.timeout_time:type_name -> google.protobuf.Timestamp + 272, // 312: openshell.v1.SandboxProvisioning.cleanup_completed_time:type_name -> google.protobuf.Timestamp + 272, // 313: openshell.v1.SandboxProvisioning.cleanup_retry_time:type_name -> google.protobuf.Timestamp + 272, // 314: openshell.v1.SandboxProvisioning.attachment_change_time:type_name -> google.protobuf.Timestamp + 272, // 315: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp + 272, // 316: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp + 126, // 317: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry.value:type_name -> openshell.v1.ProviderProfileCredential + 154, // 318: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry.value:type_name -> openshell.v1.StaticCredentialBinding + 23, // 319: openshell.v1.OpenShell.Health:input_type -> openshell.v1.HealthRequest + 25, // 320: openshell.v1.OpenShell.GetCurrentUser:input_type -> openshell.v1.GetCurrentUserRequest + 27, // 321: openshell.v1.OpenShell.GetGatewayInfo:input_type -> openshell.v1.GetGatewayInfoRequest + 51, // 322: openshell.v1.OpenShell.CreateSandbox:input_type -> openshell.v1.CreateSandboxRequest + 59, // 323: openshell.v1.OpenShell.BeginRootfsTarStaging:input_type -> openshell.v1.BeginRootfsTarStagingRequest + 61, // 324: openshell.v1.OpenShell.GetSandbox:input_type -> openshell.v1.GetSandboxRequest + 62, // 325: openshell.v1.OpenShell.ListSandboxes:input_type -> openshell.v1.ListSandboxesRequest + 52, // 326: openshell.v1.OpenShell.CreateSandboxTemplate:input_type -> openshell.v1.CreateSandboxTemplateRequest + 53, // 327: openshell.v1.OpenShell.GetSandboxTemplate:input_type -> openshell.v1.GetSandboxTemplateRequest + 54, // 328: openshell.v1.OpenShell.ListSandboxTemplates:input_type -> openshell.v1.ListSandboxTemplatesRequest + 55, // 329: openshell.v1.OpenShell.DeleteSandboxTemplate:input_type -> openshell.v1.DeleteSandboxTemplateRequest + 63, // 330: openshell.v1.OpenShell.ListSandboxProviders:input_type -> openshell.v1.ListSandboxProvidersRequest + 64, // 331: openshell.v1.OpenShell.AttachSandboxProvider:input_type -> openshell.v1.AttachSandboxProviderRequest + 65, // 332: openshell.v1.OpenShell.DetachSandboxProvider:input_type -> openshell.v1.DetachSandboxProviderRequest + 81, // 333: openshell.v1.OpenShell.GetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest + 66, // 334: openshell.v1.OpenShell.DeleteSandbox:input_type -> openshell.v1.DeleteSandboxRequest + 67, // 335: openshell.v1.OpenShell.StopSandbox:input_type -> openshell.v1.StopSandboxRequest + 68, // 336: openshell.v1.OpenShell.StartSandbox:input_type -> openshell.v1.StartSandboxRequest + 86, // 337: openshell.v1.OpenShell.CreateSshSession:input_type -> openshell.v1.CreateSshSessionRequest + 88, // 338: openshell.v1.OpenShell.ExposeService:input_type -> openshell.v1.ExposeServiceRequest + 89, // 339: openshell.v1.OpenShell.GetService:input_type -> openshell.v1.GetServiceRequest + 90, // 340: openshell.v1.OpenShell.ListServices:input_type -> openshell.v1.ListServicesRequest + 92, // 341: openshell.v1.OpenShell.DeleteService:input_type -> openshell.v1.DeleteServiceRequest + 96, // 342: openshell.v1.OpenShell.RevokeSshSession:input_type -> openshell.v1.RevokeSshSessionRequest + 98, // 343: openshell.v1.OpenShell.ExecSandbox:input_type -> openshell.v1.ExecSandboxRequest + 104, // 344: openshell.v1.OpenShell.ForwardTcp:input_type -> openshell.v1.TcpForwardFrame + 105, // 345: openshell.v1.OpenShell.ExecSandboxInteractive:input_type -> openshell.v1.ExecSandboxInput + 112, // 346: openshell.v1.OpenShell.CreateProvider:input_type -> openshell.v1.CreateProviderRequest + 113, // 347: openshell.v1.OpenShell.GetProvider:input_type -> openshell.v1.GetProviderRequest + 114, // 348: openshell.v1.OpenShell.ListProviders:input_type -> openshell.v1.ListProvidersRequest + 119, // 349: openshell.v1.OpenShell.ListProviderProfiles:input_type -> openshell.v1.ListProviderProfilesRequest + 120, // 350: openshell.v1.OpenShell.GetProviderProfile:input_type -> openshell.v1.GetProviderProfileRequest + 143, // 351: openshell.v1.OpenShell.ImportProviderProfiles:input_type -> openshell.v1.ImportProviderProfilesRequest + 145, // 352: openshell.v1.OpenShell.UpdateProviderProfiles:input_type -> openshell.v1.UpdateProviderProfilesRequest + 147, // 353: openshell.v1.OpenShell.LintProviderProfiles:input_type -> openshell.v1.LintProviderProfilesRequest + 115, // 354: openshell.v1.OpenShell.UpdateProvider:input_type -> openshell.v1.UpdateProviderRequest + 132, // 355: openshell.v1.OpenShell.GetProviderRefreshStatus:input_type -> openshell.v1.GetProviderRefreshStatusRequest + 134, // 356: openshell.v1.OpenShell.ConfigureProviderRefresh:input_type -> openshell.v1.ConfigureProviderRefreshRequest + 136, // 357: openshell.v1.OpenShell.RotateProviderCredential:input_type -> openshell.v1.RotateProviderCredentialRequest + 138, // 358: openshell.v1.OpenShell.DeleteProviderRefresh:input_type -> openshell.v1.DeleteProviderRefreshRequest + 116, // 359: openshell.v1.OpenShell.DeleteProvider:input_type -> openshell.v1.DeleteProviderRequest + 150, // 360: openshell.v1.OpenShell.DeleteProviderProfile:input_type -> openshell.v1.DeleteProviderProfileRequest + 287, // 361: openshell.v1.OpenShell.GetSandboxConfig:input_type -> openshell.sandbox.v1.GetSandboxConfigRequest + 288, // 362: openshell.v1.OpenShell.GetGatewayConfig:input_type -> openshell.sandbox.v1.GetGatewayConfigRequest + 158, // 363: openshell.v1.OpenShell.UpdateConfig:input_type -> openshell.v1.UpdateConfigRequest + 168, // 364: openshell.v1.OpenShell.GetSandboxPolicyStatus:input_type -> openshell.v1.GetSandboxPolicyStatusRequest + 170, // 365: openshell.v1.OpenShell.ListSandboxPolicies:input_type -> openshell.v1.ListSandboxPoliciesRequest + 172, // 366: openshell.v1.OpenShell.ReportPolicyStatus:input_type -> openshell.v1.ReportPolicyStatusRequest + 245, // 367: openshell.v1.OpenShell.ReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest + 83, // 368: openshell.v1.OpenShell.ReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest + 175, // 369: openshell.v1.OpenShell.ReportSandboxConfiguration:input_type -> openshell.v1.ReportSandboxConfigurationRequest + 152, // 370: openshell.v1.OpenShell.GetSandboxProviderEnvironment:input_type -> openshell.v1.GetSandboxProviderEnvironmentRequest + 156, // 371: openshell.v1.OpenShell.ExchangeProviderSubjectToken:input_type -> openshell.v1.ExchangeProviderSubjectTokenRequest + 178, // 372: openshell.v1.OpenShell.GetSandboxLogs:input_type -> openshell.v1.GetSandboxLogsRequest + 179, // 373: openshell.v1.OpenShell.PushSandboxLogs:input_type -> openshell.v1.PushSandboxLogsRequest + 182, // 374: openshell.v1.OpenShell.ConnectSupervisor:input_type -> openshell.v1.SupervisorMessage + 189, // 375: openshell.v1.OpenShell.ReportMainProcessExit:input_type -> openshell.v1.ReportMainProcessExitRequest + 191, // 376: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest + 197, // 377: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame + 199, // 378: openshell.v1.OpenShell.PeerRelay:input_type -> openshell.v1.PeerRelayFrame + 83, // 379: openshell.v1.OpenShell.PeerReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest + 245, // 380: openshell.v1.OpenShell.PeerReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest + 81, // 381: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest + 108, // 382: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest + 208, // 383: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest + 210, // 384: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest + 212, // 385: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest + 214, // 386: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest + 217, // 387: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest + 219, // 388: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest + 221, // 389: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest + 223, // 390: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest + 225, // 391: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest + 19, // 392: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest + 21, // 393: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest + 228, // 394: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest + 230, // 395: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest + 232, // 396: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest + 234, // 397: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest + 237, // 398: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest + 239, // 399: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest + 241, // 400: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest + 24, // 401: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse + 26, // 402: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse + 28, // 403: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse + 69, // 404: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse + 60, // 405: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse + 69, // 406: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse + 70, // 407: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse + 56, // 408: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 56, // 409: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 57, // 410: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse + 58, // 411: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse + 71, // 412: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse + 72, // 413: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse + 73, // 414: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse + 82, // 415: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 85, // 416: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse + 69, // 417: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse + 69, // 418: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse + 87, // 419: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse + 95, // 420: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse + 95, // 421: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse + 91, // 422: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse + 93, // 423: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse + 97, // 424: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse + 102, // 425: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent + 104, // 426: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame + 102, // 427: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent + 117, // 428: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse + 117, // 429: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse + 118, // 430: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse + 142, // 431: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse + 141, // 432: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse + 144, // 433: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse + 146, // 434: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse + 148, // 435: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse + 117, // 436: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse + 133, // 437: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse + 135, // 438: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse + 137, // 439: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse + 139, // 440: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse + 149, // 441: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse + 151, // 442: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse + 289, // 443: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse + 290, // 444: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse + 167, // 445: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse + 169, // 446: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse + 171, // 447: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse + 173, // 448: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse + 246, // 449: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 84, // 450: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 176, // 451: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse + 155, // 452: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse + 157, // 453: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse + 181, // 454: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse + 180, // 455: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse + 183, // 456: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage + 190, // 457: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse + 192, // 458: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse + 197, // 459: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame + 199, // 460: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame + 84, // 461: openshell.v1.OpenShell.PeerReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 246, // 462: openshell.v1.OpenShell.PeerReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 82, // 463: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 109, // 464: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent + 209, // 465: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse + 211, // 466: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse + 213, // 467: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse + 215, // 468: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse + 218, // 469: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse + 220, // 470: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse + 222, // 471: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse + 224, // 472: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse + 227, // 473: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse + 20, // 474: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse + 22, // 475: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse + 229, // 476: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse + 231, // 477: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse + 233, // 478: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse + 235, // 479: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse + 238, // 480: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse + 240, // 481: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse + 242, // 482: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse + 401, // [401:483] is the sub-list for method output_type + 319, // [319:401] is the sub-list for method input_type + 319, // [319:319] is the sub-list for extension type_name + 319, // [319:319] is the sub-list for extension extendee + 0, // [0:319] is the sub-list for field type_name } func init() { file_openshell_proto_init() } @@ -20066,7 +20172,7 @@ func file_openshell_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_openshell_proto_rawDesc), len(file_openshell_proto_rawDesc)), - NumEnums: 18, + NumEnums: 19, NumMessages: 253, NumExtensions: 0, NumServices: 1, diff --git a/sdk/typescript/src/client.test.ts b/sdk/typescript/src/client.test.ts index 7fef9b0db9..56dcdf3f2b 100644 --- a/sdk/typescript/src/client.test.ts +++ b/sdk/typescript/src/client.test.ts @@ -22,7 +22,7 @@ import { SCOPE_NAMES, STATUS_NAMES, } from './client.js'; -import { OpenShell, SandboxPhase, ServiceStatus } from './gen/openshell_pb.js'; +import { OpenShell, SandboxPhase, SandboxRestartPolicy, ServiceStatus } from './gen/openshell_pb.js'; import { PolicySource, SettingScope } from './gen/sandbox_pb.js'; import type { ExecInteractiveSession, ExecInteractiveSessionControl } from './index.js'; @@ -336,6 +336,20 @@ describe('create', () => { expect(created.spec?.tty).toBe(true); }); + it('sends the restart policy', async () => { + let created: { spec?: { restartPolicy?: SandboxRestartPolicy } } = {}; + const sandbox = client({ + createSandbox: (req) => { + created = req; + return readySandbox('sb', 'sb-id'); + }, + }); + + await sandbox.create({ image: 'img', restartPolicy: 'on-failure' }); + + expect(created.spec?.restartPolicy).toBe(SandboxRestartPolicy.ON_FAILURE); + }); + it('rawSpec reaches an ungated field and overrides a curated one', async () => { let created: { spec?: { @@ -842,6 +856,29 @@ describe('sandbox templates', () => { await expect(templates.delete(' ')).rejects.toMatchObject({ code: 'invalid_config' }); await expect(templates.get('missing-response')).rejects.toMatchObject({ code: 'invalid_config' }); }); + + it('maps restart controller status', async () => { + const sandbox = client({ + getSandbox: () => ({ + sandbox: { + metadata: { id: 'sb-id', name: 'sb', resourceVersion: 8n }, + status: { + phase: SandboxPhase.STARTING, + restartCount: 3, + nextRestartTime: { seconds: 1_700_000_000n, nanos: 0 }, + mainProcessStartedTime: { seconds: 1_699_999_000n, nanos: 0 }, + }, + }, + }), + }); + + await expect(sandbox.get('sb')).resolves.toMatchObject({ + phase: 'starting', + restartCount: 3, + nextRestartAtMs: 1_700_000_000_000, + mainProcessStartedAtMs: 1_699_999_000_000, + }); + }); }); describe('waits', () => { diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts index 93ad8ba6d1..2f26a71f91 100644 --- a/sdk/typescript/src/client.ts +++ b/sdk/typescript/src/client.ts @@ -23,6 +23,7 @@ import { type ExecSandboxInputSchema, OpenShell, SandboxPhase, + SandboxRestartPolicy, type SandboxSpecSchema, type SandboxWorkloadTemplateSchema, ServiceStatus, @@ -84,6 +85,9 @@ export type SandboxPhaseName = | 'starting' | 'completed'; +/** Restart behavior after the canonical main process exits. */ +export type SandboxRestartPolicyName = 'never' | 'on-failure' | 'always'; + /** Lowercase mirror of the generated `ServiceStatus` enum. Hand-maintained. */ export type HealthStatus = 'unspecified' | 'healthy' | 'degraded' | 'unhealthy'; @@ -141,6 +145,8 @@ export interface SandboxSpec { tty?: boolean; /** Loopback HTTP services to expose when the sandbox is created. */ serviceExposures?: ServiceExposure[]; + /** Restart behavior after the canonical main process exits. */ + restartPolicy?: SandboxRestartPolicyName; /** * Create-time sandbox policy (the safety boundary). Sandbox-scoped * `setPolicy` cannot introduce static fields later, so express filesystem, @@ -197,6 +203,9 @@ export interface SandboxRef { createdFromWorkloadTemplate?: SandboxWorkloadTemplateProvenance; /** Service URLs returned by creation, keyed by service name. */ serviceUrls: Record; + restartCount: number; + nextRestartAtMs?: number; + mainProcessStartedAtMs?: number; } export interface SandboxWorkloadTemplateProvenance { @@ -472,6 +481,17 @@ export const POLICY_SOURCE_NAMES: Record = { function phaseName(p: SandboxPhase): SandboxPhaseName { return PHASE_NAMES[p] ?? 'unspecified'; } + +function restartPolicyValue(policy: SandboxRestartPolicyName | undefined): SandboxRestartPolicy { + switch (policy) { + case 'on-failure': + return SandboxRestartPolicy.ON_FAILURE; + case 'always': + return SandboxRestartPolicy.ALWAYS; + default: + return SandboxRestartPolicy.NEVER; + } +} function statusName(s: ServiceStatus): HealthStatus { return STATUS_NAMES[s] ?? 'unspecified'; } @@ -488,6 +508,8 @@ function sandboxRef(sandbox: Sandbox | undefined, serviceUrls: Record` for the policy, exit code, restart count, and +next restart time. `Starting` can mean the gateway is waiting for backoff or +for a replacement supervisor. Check gateway logs for stop or start errors if +the deadline passes without a transition to `Ready`. Docker, Podman, +Kubernetes, and VM native restart policies remain disabled; the gateway owns +the replacement. + ### Step 5: Check Podman-Backed Gateways ```bash diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index 818d12a269..4db30ebac0 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -277,6 +277,7 @@ Key flags: - `--label KEY=VALUE`: Add labels for later selection (repeatable) - `--env KEY=VALUE`: Set non-secret sandbox environment variables (repeatable); use `--provider` for credentials - `--tty`: Allocate a retained PTY for the canonical main process +- `--restart-policy never|on-failure|always`: Select gateway-owned main-process restart behavior; `never` is the default - `--approval-mode manual|auto`: Control handling of agent-authored policy proposals; `manual` is the default - `--upload [:]`: Upload local files into the container working directory or an explicit destination - `--no-git-ignore`: Disable `.gitignore` filtering for uploads @@ -377,9 +378,13 @@ openshell sandbox ssh-config my-sandbox >> ~/.ssh/config ``` If `connect` reports `canonical main process already finished`, inspect the -result with `sandbox get`. A pending -foreground attachment can still retrieve retained output in `Completed` or -`Error`; phase alone does not determine whether attachment is available. +result with `sandbox get`. A pending foreground attachment can still retrieve +retained output in `Completed` or `Error`; phase alone does not determine +whether attachment is available. A nonzero main-process exit under +`on-failure`, or any exit under `always`, +moves the sandbox to `Starting` during backoff and resource replacement. Connect +and exec commands resume after the new supervisor session makes it `Ready`. +An explicit `sandbox stop` cancels a pending restart. ### Upload and download files From 2fe5a0e19ce55e26454223f6ca45549e843b36e3 Mon Sep 17 00:00:00 2001 From: Divesh <47188680+FrostGod@users.noreply.github.com> Date: Tue, 29 Sep 2026 04:47:12 +0000 Subject: [PATCH 2/3] perf(kubernetes): use a TCP readiness probe for the supervisor (#3700) - Kubernetes now checks supervisor readiness by connecting to TCP port 5501 - Stop starting a supervisor process in every sandbox each second - The supervisor opens the port only while its gateway session is up - Accept IPv4 and IPv6 probes, even when net.ipv6.bindv6only is set - Keep the health socket for Docker, Podman, and debugging - Add tests and update the docs Signed-off-by: divesh --- Cargo.lock | 1 + architecture/build.md | 8 +- .../src/sandbox_runtime.rs | 44 ++-- crates/openshell-supervisor/Cargo.toml | 1 + crates/openshell-supervisor/src/lib.rs | 197 ++++++++++++++---- crates/openshell-supervisor/src/main.rs | 6 + docs/kubernetes/setup.mdx | 2 + 7 files changed, 195 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db4da3dfd2..4c63554b91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4771,6 +4771,7 @@ dependencies = [ "rustls", "serde", "serde_json", + "socket2", "temp-env", "tempfile", "tokio", diff --git a/architecture/build.md b/architecture/build.md index 556a3a0094..89174a7317 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -210,9 +210,11 @@ Runtime layout: `GLIBC_2.28`. Image defaults remain UID 0 and working directory `/`; compute drivers set the runtime identity and writable mounts. Docker stages private files with the same numeric identity as the supervisor so archive uploads - preserve access regardless of the base image's default user. Health probes execute - the supervisor binary directly. Base updates require refreshing the - multi-architecture digest and rebuilding the image. + preserve access regardless of the base image's default user. Docker and Podman + health checks execute the supervisor binary directly. Kubernetes uses a kubelet + `tcpSocket` readiness probe on supervisor port 5501 instead, so the probe does + not start a process in every sandbox each period. Base updates require + refreshing the multi-architecture digest and rebuilding the image. Gateway image builds bake the corresponding supervisor image tag into the gateway binary so Docker sandboxes do not depend on `:latest` by default. diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 0588bbb95c..abee5eaf03 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -8,10 +8,11 @@ use std::path::Path; use k8s_openapi::ByteString; use k8s_openapi::api::core::v1::{ - CSIVolumeSource, Capabilities, Container, EmptyDirVolumeSource, EnvVar, ExecAction, KeyToPath, + CSIVolumeSource, Capabilities, Container, EmptyDirVolumeSource, EnvVar, KeyToPath, LocalObjectReference, Pod, PodSchedulingGate, PodSecurityContext, PodSpec, Probe, ProjectedVolumeSource, Secret, SecretVolumeSource, SecurityContext, Service, - ServiceAccountTokenProjection, ServicePort, ServiceSpec, Volume, VolumeMount, VolumeProjection, + ServiceAccountTokenProjection, ServicePort, ServiceSpec, TCPSocketAction, Volume, VolumeMount, + VolumeProjection, }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; @@ -58,6 +59,9 @@ pub const CLIENT_TLS_CA_PATH: &str = "/.openshell/supervisor/client-ca.crt"; pub const CLIENT_TLS_CERTIFICATE_PATH: &str = "/.openshell/supervisor/client-tls.crt"; pub const CLIENT_TLS_PRIVATE_KEY_PATH: &str = "/.openshell/supervisor/client-tls.key"; pub const CONTROL_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock"; +/// Kubelet `tcpSocket` readiness port. An exec probe would start a supervisor +/// process in every sandbox on every period. +pub const CONTROL_HEALTH_PORT: u16 = 5501; pub const NAMESPACE_WORKLOAD_POLICY_NAME: &str = "openshell-sandbox-workloads"; pub const NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME: &str = "openshell-sandbox-supervisors"; pub const SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS: i64 = 30; @@ -334,6 +338,8 @@ pub fn supervisor_pod( "/sandbox".to_string(), "--health-socket-path".to_string(), CONTROL_HEALTH_SOCKET_PATH.to_string(), + "--health-port".to_string(), + CONTROL_HEALTH_PORT.to_string(), ]; if let Some(url) = https_proxy { command.extend(["--upstream-proxy".to_string(), url.to_string()]); @@ -411,13 +417,9 @@ pub fn supervisor_pod( termination_message_policy: Some("FallbackToLogsOnError".to_string()), env: Some(environment), readiness_probe: Some(Probe { - exec: Some(ExecAction { - command: Some(vec![ - "/openshell-supervisor".to_string(), - "health".to_string(), - "--socket".to_string(), - CONTROL_HEALTH_SOCKET_PATH.to_string(), - ]), + tcp_socket: Some(TCPSocketAction { + port: IntOrString::Int(i32::from(CONTROL_HEALTH_PORT)), + ..Default::default() }), period_seconds: Some(1), failure_threshold: Some(3), @@ -984,18 +986,14 @@ mod tests { .and_then(|capabilities| capabilities.drop.as_ref()), Some(&vec!["ALL".to_string()]) ); + let probe = container.readiness_probe.as_ref().expect("readiness probe"); + assert!( + probe.exec.is_none(), + "exec probes spawn a process per period" + ); assert_eq!( - container - .readiness_probe - .as_ref() - .and_then(|probe| probe.exec.as_ref()) - .and_then(|exec| exec.command.as_ref()), - Some(&vec![ - "/openshell-supervisor".to_string(), - "health".to_string(), - "--socket".to_string(), - CONTROL_HEALTH_SOCKET_PATH.to_string(), - ]) + probe.tcp_socket.as_ref().map(|tcp| &tcp.port), + Some(&IntOrString::Int(i32::from(CONTROL_HEALTH_PORT))) ); let command = container.command.as_ref().unwrap(); assert!( @@ -1003,6 +1001,12 @@ mod tests { .windows(2) .any(|args| args == ["--health-socket-path", CONTROL_HEALTH_SOCKET_PATH]) ); + let health_port = CONTROL_HEALTH_PORT.to_string(); + assert!( + command + .windows(2) + .any(|args| args == ["--health-port", health_port.as_str()]) + ); let env = container.env.as_ref().unwrap(); let env_value = |name: &str| { env.iter() diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index d4fb9cee68..483dd50bff 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -35,6 +35,7 @@ rustls = { workspace = true } rustix = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +socket2 = { workspace = true } tempfile = "3" tokio = { workspace = true } tonic = { workspace = true, features = ["channel", "tls-native-roots"] } diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 8dc3f97600..cdda9a61c4 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -90,32 +90,97 @@ where } } +/// Where the supervisor publishes readiness. The listener exists only while +/// the supervisor session is ready, so a successful connect means ready. +#[derive(Clone, Debug)] +enum ReadinessEndpoint { + Unix(std::path::PathBuf), + /// Wildcard TCP port reachable from the node, for kubelet `tcpSocket` probes. + Tcp(u16), +} + +enum ReadinessListener { + Unix(tokio::net::UnixListener), + Tcp(tokio::net::TcpListener), +} + +impl ReadinessEndpoint { + fn bind(&self) -> Result { + match self { + Self::Unix(path) => { + prepare_control_readiness_path(path)?; + tokio::net::UnixListener::bind(path) + .map(ReadinessListener::Unix) + .into_diagnostic() + .wrap_err_with(|| { + format!("bind supervisor readiness socket on {}", path.display()) + }) + } + Self::Tcp(port) => bind_readiness_tcp(*port) + .and_then(tokio::net::TcpListener::from_std) + .map(ReadinessListener::Tcp) + .into_diagnostic() + .wrap_err_with(|| format!("bind supervisor readiness listener on port {port}")), + } + } + + fn remove(&self) { + if let Self::Unix(path) = self { + let _ = std::fs::remove_file(path); + } + } +} + +/// Falls back to IPv4 when the network namespace has IPv6 disabled. +fn bind_readiness_tcp(port: u16) -> std::io::Result { + use socket2::{Domain, Socket, Type}; + + let bind = |domain: Domain, address: std::net::SocketAddr| { + let socket = Socket::new(domain, Type::STREAM, None)?; + if domain == Domain::IPV6 { + socket.set_only_v6(false)?; + } + socket.set_reuse_address(true)?; + socket.bind(&address.into())?; + socket.listen(128)?; + socket.set_nonblocking(true)?; + Ok::<_, std::io::Error>(std::net::TcpListener::from(socket)) + }; + bind(Domain::IPV6, (std::net::Ipv6Addr::UNSPECIFIED, port).into()) + .or_else(|_| bind(Domain::IPV4, (std::net::Ipv4Addr::UNSPECIFIED, port).into())) +} + +impl ReadinessListener { + async fn accept(&self) -> std::io::Result<()> { + match self { + Self::Unix(listener) => listener.accept().await.map(drop), + Self::Tcp(listener) => listener.accept().await.map(drop), + } + } +} + struct ControlReadiness { task: tokio::task::JoinHandle<()>, - path: std::path::PathBuf, + endpoint: ReadinessEndpoint, } impl ControlReadiness { fn start( - path: std::path::PathBuf, + endpoint: ReadinessEndpoint, mut session_readiness: Option>, ) -> Result { - prepare_control_readiness_path(&path)?; + if let ReadinessEndpoint::Unix(path) = &endpoint { + prepare_control_readiness_path(path)?; + } let listener = if session_readiness .as_ref() .is_some_and(|readiness| !*readiness.borrow()) { None } else { - Some( - tokio::net::UnixListener::bind(&path) - .into_diagnostic() - .wrap_err_with(|| { - format!("bind supervisor readiness socket on {}", path.display()) - })?, - ) + Some(endpoint.bind()?) }; - let task_path = path.clone(); + let task_endpoint = endpoint.clone(); let task = tokio::spawn(async move { let mut listener = listener; loop { @@ -125,7 +190,7 @@ impl ControlReadiness { if listener.is_none() || session_unready { if session_unready { listener.take(); - let _ = std::fs::remove_file(&task_path); + task_endpoint.remove(); let Some(readiness) = session_readiness.as_mut() else { break; }; @@ -133,16 +198,7 @@ impl ControlReadiness { break; } } - match prepare_control_readiness_path(&task_path).and_then(|()| { - tokio::net::UnixListener::bind(&task_path) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "rebind supervisor readiness socket on {}", - task_path.display() - ) - }) - }) { + match task_endpoint.bind() { Ok(rebound) => listener = Some(rebound), Err(error) => { tracing::warn!(%error, "control-mode readiness rebind failed; retrying"); @@ -159,7 +215,7 @@ impl ControlReadiness { if let Some(readiness) = session_readiness.as_mut() { tokio::select! { accepted = active_listener.accept() => match accepted { - Ok((stream, _)) => drop(stream), + Ok(()) => {} Err(error) => { tracing::warn!(%error, "control-mode readiness accept failed; retrying"); tokio::time::sleep(Duration::from_millis(100)).await; @@ -173,7 +229,7 @@ impl ControlReadiness { } } else { match active_listener.accept().await { - Ok((stream, _)) => drop(stream), + Ok(()) => {} Err(error) => { tracing::warn!(%error, "control-mode readiness accept failed; retrying"); tokio::time::sleep(Duration::from_millis(100)).await; @@ -181,9 +237,9 @@ impl ControlReadiness { } } } - let _ = std::fs::remove_file(&task_path); + task_endpoint.remove(); }); - Ok(Self { task, path }) + Ok(Self { task, endpoint }) } } @@ -228,7 +284,7 @@ fn prepare_control_readiness_path(path: &std::path::Path) -> Result<()> { impl Drop for ControlReadiness { fn drop(&mut self) { self.task.abort(); - let _ = std::fs::remove_file(&self.path); + self.endpoint.remove(); } } @@ -571,6 +627,7 @@ pub async fn run_sandbox( policy_data: Option, ssh_socket_path: Option, health_socket_path: Option, + health_port: Option, ocsf_enabled: Arc, ocsf_schema_version: Arc>, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, @@ -1125,14 +1182,12 @@ pub async fn run_sandbox( running.exec(), ) }); - let mut control_readiness = if let Some(path) = health_socket_path { - Some(ControlReadiness::start( - path, - boundary_access.session_readiness(), - )?) - } else { - None - }; + let mut control_readiness = health_socket_path + .map(ReadinessEndpoint::Unix) + .into_iter() + .chain(health_port.map(ReadinessEndpoint::Tcp)) + .map(|endpoint| ControlReadiness::start(endpoint, boundary_access.session_readiness())) + .collect::>>()?; let instance_id = boundary_access.instance_id().to_string(); let wait_agent = agent.clone(); let shutdown_requested = wait_for_control_shutdown_signal(); @@ -1186,7 +1241,7 @@ pub async fn run_sandbox( } }; if !retain_access { - control_readiness.take(); + control_readiness.clear(); } boundary_access .publish_main_exit(exit_code, await_main_process_attachment) @@ -1231,7 +1286,7 @@ pub async fn run_sandbox( } if completion_cancelled { retain_access = false; - control_readiness.take(); + control_readiness.clear(); } if retain_access { info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); @@ -4744,8 +4799,8 @@ mod tests { async fn control_readiness_exists_only_while_guard_is_live() { let root = tempfile::tempdir().unwrap(); let path = root.path().join("health.sock"); - let readiness = - ControlReadiness::start(path.clone(), None).expect("start readiness listener"); + let readiness = ControlReadiness::start(ReadinessEndpoint::Unix(path.clone()), None) + .expect("start readiness listener"); check_control_readiness(&path).expect("running supervisor accepts readiness probes"); drop(readiness); @@ -4758,8 +4813,9 @@ mod tests { let root = tempfile::tempdir().unwrap(); let path = root.path().join("health.sock"); let (session_tx, session_rx) = tokio::sync::watch::channel(false); - let _readiness = ControlReadiness::start(path.clone(), Some(session_rx)) - .expect("start readiness listener"); + let _readiness = + ControlReadiness::start(ReadinessEndpoint::Unix(path.clone()), Some(session_rx)) + .expect("start readiness listener"); assert!(check_control_readiness(&path).is_err()); session_tx.send_replace(true); @@ -4790,6 +4846,65 @@ mod tests { .expect("replacement session restores readiness socket"); } + #[test] + fn tcp_readiness_listener_accepts_ipv4_regardless_of_bindv6only() { + let port = std::net::TcpListener::bind("127.0.0.1:0") + .and_then(|reserved| reserved.local_addr()) + .expect("reserve loopback port") + .port(); + let listener = bind_readiness_tcp(port).expect("bind readiness listener"); + if listener.local_addr().expect("local address").is_ipv6() { + assert!( + !socket2::SockRef::from(&listener) + .only_v6() + .expect("read IPV6_V6ONLY"), + "net.ipv6.bindv6only=1 must not make the wildcard listener IPv6-only" + ); + } + std::net::TcpStream::connect(("127.0.0.1", port)) + .expect("IPv4 kubelet probe reaches the readiness listener"); + } + + #[tokio::test] + async fn tcp_control_readiness_tracks_supervisor_session() { + let port = std::net::TcpListener::bind("127.0.0.1:0") + .and_then(|reserved| reserved.local_addr()) + .expect("reserve loopback port") + .port(); + let (session_tx, session_rx) = tokio::sync::watch::channel(true); + let readiness = ControlReadiness::start(ReadinessEndpoint::Tcp(port), Some(session_rx)) + .expect("start TCP readiness listener"); + let connects = || std::net::TcpStream::connect(("127.0.0.1", port)).is_ok(); + assert!(connects(), "accepted session is ready"); + + session_tx.send_replace(false); + timeout(Duration::from_secs(1), async { + while connects() { + tokio::task::yield_now().await; + } + }) + .await + .expect("lost session closes readiness listener"); + + session_tx.send_replace(true); + timeout(Duration::from_secs(1), async { + while !connects() { + tokio::task::yield_now().await; + } + }) + .await + .expect("replacement session reopens readiness listener"); + + drop(readiness); + timeout(Duration::from_secs(1), async { + while connects() { + tokio::task::yield_now().await; + } + }) + .await + .expect("dropped guard closes readiness listener"); + } + #[test] fn control_readiness_rejects_relative_path() { let error = prepare_control_readiness_path(std::path::Path::new("health.sock")) diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index df10d6b32a..4bfb469891 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -84,6 +84,10 @@ struct Args { #[arg(long, env = "OPENSHELL_HEALTH_SOCKET_PATH")] health_socket_path: Option, + /// TCP port that accepts connections only while the supervisor is ready. + #[arg(long, env = "OPENSHELL_HEALTH_PORT")] + health_port: Option, + #[arg(long)] upstream_proxy: Option, @@ -234,6 +238,7 @@ fn validate_role_arguments(args: &Args) -> Result<()> { || args.openshell_endpoint.is_some() || args.ssh_socket_path.is_some() || args.health_socket_path.is_some() + || args.health_port.is_some() || args.main_exit_marker.is_some() || args.parent_liveness_fd.is_some() { @@ -439,6 +444,7 @@ fn main() -> Result<()> { args.policy_data, args.ssh_socket_path, args.health_socket_path, + args.health_port, ocsf_enabled, ocsf_schema_version, upstream_proxy_args, diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index b16065aa61..2b39ddf223 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -485,6 +485,8 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency - `startupProbe` and `livenessProbe` use `/healthz`. - `readinessProbe` uses `/readyz`, which reflects the latest result of an in-process background database check. +Each sandbox supervisor Pod uses a `tcpSocket` readiness probe on port 5501. The supervisor accepts connections on that port only while its gateway session is ready. If you add a default-deny ingress policy to sandbox namespaces, allow TCP port 5501 from the node so kubelet can reach supervisor Pods. + ## Next Steps - To run multiple gateway replicas, refer to [High Availability](/kubernetes/high-availability). From 9cb72baa2e61a1b5f12407e6e82da7fdba0aa722 Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Tue, 29 Sep 2026 05:15:02 +0000 Subject: [PATCH 3/3] feat(docker): support corporate proxy CA bundles (#3549) * feat(docker): support corporate proxy CA bundles Closes #3545 Validate and stage operator-owned proxy CA bundles for Docker supervisors, add corporate proxy E2E coverage, and document the trust contract. Signed-off-by: Philippe Martin * fix(docker): validate proxy config on startup Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * test(docker): use the E2E workload image for proxy tests Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * test(docker): generate strict corporate proxy certificates Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * test(docker): surface intercepted TLS fixture errors Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * test(docker): drain buffered TLS proxy data Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * test(docker): relay intercepted HTTP deterministically Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --------- Signed-off-by: Philippe Martin Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> Co-authored-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 9 +- crates/openshell-driver-docker/README.md | 26 + crates/openshell-driver-docker/src/lib.rs | 82 +- crates/openshell-driver-docker/src/tests.rs | 164 +++ docs/how-it-works/gateways/configuration.mdx | 31 + docs/how-it-works/sandboxes/runtimes.mdx | 20 + e2e/rust/Cargo.toml | 5 + e2e/rust/tests/docker_corporate_proxy.rs | 1180 ++++++++++++++++++ skills/debug-openshell-cluster/SKILL.md | 28 + tasks/scripts/gateway-docker.sh | 3 + 10 files changed, 1533 insertions(+), 15 deletions(-) create mode 100644 e2e/rust/tests/docker_corporate_proxy.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index a704b25341..8b4d30efc7 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -585,8 +585,13 @@ cross the authenticated vsock channel. A gateway-host proxy is addressed as The Docker driver runs `openshell-supervisor` in a separate companion container. Its private named volume contains supervisor bootstrap and channel material. -The workload container receives only `openshell-sandbox`, public interception -CA material, and the other sandbox half of the authenticated channel. +The driver bounded-reads operator proxy credentials and CA bundles into that +supervisor-only volume and passes fixed container paths to the supervisor; it +never exposes gateway-host paths to sandbox-controlled configuration. The +corporate CA extends supervisor upstream trust and is folded into the public +combined trust bundle generated for workload processes. The workload container +receives only `openshell-sandbox`, public interception and combined CA +material, and the other sandbox half of the authenticated channel. For Kubernetes, the operator configures a Secret name and key rather than a gateway-host file path. Kubernetes projects that Secret only into the separate diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index a25c455eea..ee329f8dea 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -50,6 +50,32 @@ through the Docker archive API. No workload launch depends on a host bind mount or a tool supplied by the workload image, so the same path works with local, remote, and VM-backed Docker daemons. +## Corporate Proxy Egress + +`[openshell.drivers.docker]` accepts the operator-owned corporate proxy fields +`https_proxy`, `no_proxy`, `proxy_auth_file`, +`proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, and +`proxy_ca_bundle`. Sandbox environment, image contents, and per-sandbox driver +configuration cannot select or override them. + +The driver validates the proxy URL and cross-field relationships at gateway +startup. It bounded-reads proxy credentials and the PEM CA bundle before use. +Missing, unreadable, empty, oversized, malformed, or certificate-free CA files +fail closed. A CA bundle requires `https_proxy`, although the proxy URL may be +`http://` when the proxy intercepts destination TLS. + +For each supervisor launch, Docker copies the credential and CA contents into +the existing supervisor-only named volume. It passes only the fixed container +paths `/.openshell/supervisor/upstream-proxy-auth` and +`/.openshell/supervisor/upstream-proxy-ca-bundle.pem` to the supervisor. It +does not bind-mount the gateway-host files, which preserves remote-daemon +support and keeps host paths out of workload container metadata. + +The supervisor trusts the configured CA for its TLS connection to an HTTPS +proxy and for destination certificates re-signed by a TLS-intercepting proxy. +It also includes the corporate root in the generated combined trust bundle +used by workload processes. + ## Identity and Workspace Before creating the workload, the driver pins the image ID and reads its diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 1f4c91f5e4..d4a1f2f71d 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -107,6 +107,8 @@ const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/serve const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/supervisor"; const SUPERVISOR_PROXY_AUTH_MOUNT_PATH: &str = "/.openshell/supervisor/upstream-proxy-auth"; +const SUPERVISOR_PROXY_CA_BUNDLE_MOUNT_PATH: &str = + "/.openshell/supervisor/upstream-proxy-ca-bundle.pem"; const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR; const DRIVER_ADMITTED_BACKEND: &str = openshell_sandbox_backend::BACKEND_NAME; @@ -218,6 +220,10 @@ pub struct DockerComputeConfig { #[serde(flatten)] pub upstream_proxy: UpstreamProxyConfig, + /// Gateway-host PEM CA bundle trusted for the corporate proxy and for + /// server certificates re-signed by a TLS-intercepting proxy. + pub proxy_ca_bundle: Option, + /// Host UNIX socket projected into the supervisor for provider identity. pub provider_spiffe_workload_api_socket: Option, @@ -242,6 +248,7 @@ impl DockerComputeConfig { validate_image_pull_policy(self.image_pull_policy)?; self.upstream_proxy.validate().map_err(Error::config)?; validate_docker_proxy_auth_file(&self.upstream_proxy)?; + validate_docker_proxy_ca_bundle(self)?; if let Some(socket) = self.provider_spiffe_workload_api_socket.as_deref() { openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) .map_err(Error::config)?; @@ -276,6 +283,7 @@ impl Default for DockerComputeConfig { sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, upstream_proxy: UpstreamProxyConfig::default(), + proxy_ca_bundle: None, provider_spiffe_workload_api_socket: None, app_armor_profile: None, } @@ -307,6 +315,7 @@ struct DockerDriverRuntimeConfig { sandbox_pids_limit: Option, enable_bind_mounts: bool, upstream_proxy: UpstreamProxyConfig, + proxy_ca_bundle: Option, provider_spiffe_workload_api_socket: Option, app_armor_profile: Option, } @@ -837,10 +846,7 @@ impl DockerComputeDriver { gateway_log_level: &str, docker_config: &DockerComputeConfig, ) -> CoreResult { - docker_config - .resource_admission - .validate() - .map_err(Error::config)?; + docker_config.validate_configuration(gateway_bind_address)?; let socket_path = docker_config .socket_path .clone() @@ -873,15 +879,8 @@ impl DockerComputeDriver { cdi_supported, wsl_all_gpu_fallback_enabled, }; - validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; - validate_image_pull_policy(docker_config.image_pull_policy)?; validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; let gateway_port = gateway_bind_address.port(); - if gateway_port == 0 { - return Err(Error::config( - "docker compute driver requires a fixed non-zero gateway bind port", - )); - } let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { docker_config.grpc_endpoint = default_docker_supervisor_grpc_endpoint( @@ -947,6 +946,7 @@ impl DockerComputeDriver { allow_driver_config: docker_config.allow_driver_config, resource_admission: docker_config.resource_admission.clone(), upstream_proxy: docker_config.upstream_proxy.clone(), + proxy_ca_bundle: docker_config.proxy_ca_bundle.clone(), provider_spiffe_workload_api_socket: docker_config .provider_spiffe_workload_api_socket .clone(), @@ -4713,11 +4713,35 @@ async fn docker_supervisor_bundle_archive( &contents, )?; } + append_docker_proxy_ca_bundle(&mut archive, config.proxy_ca_bundle.as_deref())?; archive .into_inner() .map_err(|error| Status::internal(format!("finish Docker supervisor archive: {error}"))) } +fn append_docker_proxy_ca_bundle( + archive: &mut tar::Builder>, + path: Option<&Path>, +) -> Result<(), Status> { + let Some(path) = path else { + return Ok(()); + }; + let path = path + .to_str() + .ok_or_else(|| Status::failed_precondition("proxy_ca_bundle must be valid UTF-8"))?; + let contents = + openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file(path, "proxy_ca_bundle") + .map_err(Status::failed_precondition)?; + append_docker_archive_file( + archive, + "upstream-proxy-ca-bundle.pem", + 0o644, + SUPERVISOR_UID, + SUPERVISOR_GID, + contents.as_bytes(), + ) +} + async fn refresh_docker_boundary_authentication( sandbox_id: &str, config: &DockerDriverRuntimeConfig, @@ -5120,7 +5144,10 @@ async fn spawn_docker_control_process( workspace_root, format!("--health-socket-path={SUPERVISOR_HEALTH_SOCKET_PATH}"), ]; - command.extend(docker_upstream_proxy_cli_args(&config.upstream_proxy)); + command.extend(docker_upstream_proxy_cli_args( + &config.upstream_proxy, + config.proxy_ca_bundle.is_some(), + )); let mut supervisor_mounts = vec![ Mount { target: Some(BOUNDARY_MOUNT_PATH.to_string()), @@ -5871,7 +5898,30 @@ fn validate_docker_proxy_auth_file(config: &UpstreamProxyConfig) -> CoreResult<( Ok(()) } -fn docker_upstream_proxy_cli_args(config: &UpstreamProxyConfig) -> Vec { +fn validate_docker_proxy_ca_bundle(config: &DockerComputeConfig) -> CoreResult<()> { + let Some(path) = config.proxy_ca_bundle.as_ref() else { + return Ok(()); + }; + if path.as_os_str().is_empty() { + return Err(Error::config("proxy_ca_bundle must not be empty when set")); + } + if config.upstream_proxy.https_proxy.is_none() { + return Err(Error::config( + "proxy_ca_bundle is set but no https_proxy is configured", + )); + } + let path = path + .to_str() + .ok_or_else(|| Error::config("proxy_ca_bundle must be valid UTF-8"))?; + openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file(path, "proxy_ca_bundle") + .map_err(Error::config)?; + Ok(()) +} + +fn docker_upstream_proxy_cli_args( + config: &UpstreamProxyConfig, + proxy_ca_bundle_configured: bool, +) -> Vec { let mut args = Vec::new(); if let Some(url) = config.https_proxy.as_ref() { args.extend(["--upstream-proxy".to_string(), url.clone()]); @@ -5891,6 +5941,12 @@ fn docker_upstream_proxy_cli_args(config: &UpstreamProxyConfig) -> Vec { if config.proxy_connect_by_hostname == Some(true) { args.push("--upstream-proxy-connect-by-hostname".to_string()); } + if proxy_ca_bundle_configured { + args.extend([ + "--upstream-proxy-ca-bundle".to_string(), + SUPERVISOR_PROXY_CA_BUNDLE_MOUNT_PATH.to_string(), + ]); + } args } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index c5036f4547..bc747312c3 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -20,6 +20,7 @@ use openshell_core::proto::compute::v1::{ ResourceRequirements, WorkloadIdentityRequest, }; use std::fs; +use std::io::Read as _; use std::sync::Arc; use tempfile::TempDir; @@ -197,11 +198,174 @@ fn runtime_config() -> DockerDriverRuntimeConfig { sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, upstream_proxy: UpstreamProxyConfig::default(), + proxy_ca_bundle: None, provider_spiffe_workload_api_socket: None, app_armor_profile: Some(AppArmorProfile::Unconfined), } } +fn write_test_proxy_ca_bundle(directory: &TempDir) -> PathBuf { + let tls = generate_sandbox_tls_material(openshell_core::SandboxSessionId::new()) + .expect("generate test proxy CA"); + let path = directory.path().join("proxy-ca.pem"); + fs::write(&path, tls.trust_anchor_pem).expect("write test proxy CA"); + path +} + +#[test] +fn docker_config_parses_operator_proxy_ca_bundle() { + let config: DockerComputeConfig = toml::from_str( + r#" +https_proxy = "https://proxy.corp.example:8443" +proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +"#, + ) + .expect("parse Docker proxy CA configuration"); + + assert_eq!( + config.proxy_ca_bundle, + Some(PathBuf::from("/etc/openshell/tls/proxy-ca.pem")) + ); +} + +#[test] +fn docker_proxy_ca_bundle_validation_is_fail_closed() { + let directory = TempDir::new().expect("create CA directory"); + let ca_bundle = write_test_proxy_ca_bundle(&directory); + let gateway_bind_address = "127.0.0.1:17670".parse().unwrap(); + + let mut valid = DockerComputeConfig::default(); + valid.upstream_proxy.https_proxy = Some("http://proxy.corp.example:8080".to_string()); + valid.proxy_ca_bundle = Some(ca_bundle); + valid + .validate_configuration(gateway_bind_address) + .expect("a valid CA bundle is accepted with an HTTP interception proxy"); + + let mut without_proxy = valid.clone(); + without_proxy.upstream_proxy.https_proxy = None; + let error = without_proxy + .validate_configuration(gateway_bind_address) + .expect_err("a CA bundle without a proxy must fail"); + assert!(error.to_string().contains("proxy_ca_bundle"), "{error}"); + assert!(error.to_string().contains("https_proxy"), "{error}"); + + let mut empty_path = valid.clone(); + empty_path.proxy_ca_bundle = Some(PathBuf::new()); + let error = empty_path + .validate_configuration(gateway_bind_address) + .expect_err("an empty CA bundle path must fail"); + assert!(error.to_string().contains("must not be empty"), "{error}"); + + let mut missing = valid.clone(); + missing.proxy_ca_bundle = Some(directory.path().join("missing.pem")); + let error = missing + .validate_configuration(gateway_bind_address) + .expect_err("a missing CA bundle must fail"); + assert!(error.to_string().contains("could not be read"), "{error}"); + + let malformed_path = directory.path().join("malformed.pem"); + fs::write(&malformed_path, "not a certificate\n").unwrap(); + let mut malformed = valid; + malformed.proxy_ca_bundle = Some(malformed_path); + let error = malformed + .validate_configuration(gateway_bind_address) + .expect_err("a certificate-free CA bundle must fail"); + assert!(error.to_string().contains("no PEM certificate"), "{error}"); +} + +#[tokio::test] +async fn docker_constructor_rejects_proxy_ca_bundle_without_proxy() { + let directory = TempDir::new().expect("create CA directory"); + let mut config = DockerComputeConfig { + socket_path: Some(directory.path().join("unused-docker.sock")), + proxy_ca_bundle: Some(write_test_proxy_ca_bundle(&directory)), + ..DockerComputeConfig::default() + }; + config.upstream_proxy.https_proxy = None; + + let Err(error) = + DockerComputeDriver::new("127.0.0.1:17670".parse().unwrap(), "info", &config).await + else { + panic!("constructor must reject incoherent proxy CA configuration before Docker I/O"); + }; + + assert!(error.to_string().contains("proxy_ca_bundle"), "{error}"); + assert!(error.to_string().contains("https_proxy"), "{error}"); +} + +#[test] +fn docker_proxy_ca_bundle_uses_fixed_supervisor_path() { + let proxy = UpstreamProxyConfig { + https_proxy: Some("https://proxy.corp.example:8443".to_string()), + ..UpstreamProxyConfig::default() + }; + + let args = docker_upstream_proxy_cli_args(&proxy, true); + let option = args + .iter() + .position(|arg| arg == "--upstream-proxy-ca-bundle") + .expect("proxy CA option"); + assert_eq!( + args.get(option + 1).map(String::as_str), + Some(SUPERVISOR_PROXY_CA_BUNDLE_MOUNT_PATH) + ); + assert!( + !args.iter().any(|arg| arg.contains("/etc/openshell/tls")), + "gateway-host paths must not appear in supervisor argv: {args:?}" + ); + + let args = docker_upstream_proxy_cli_args(&proxy, false); + assert!(!args.iter().any(|arg| arg == "--upstream-proxy-ca-bundle")); +} + +#[test] +fn docker_proxy_ca_bundle_is_staged_in_supervisor_archive() { + let directory = TempDir::new().expect("create CA directory"); + let ca_bundle = write_test_proxy_ca_bundle(&directory); + let expected = fs::read_to_string(&ca_bundle).unwrap(); + let mut builder = tar::Builder::new(Vec::new()); + + append_docker_proxy_ca_bundle(&mut builder, Some(&ca_bundle)).expect("append proxy CA bundle"); + let archive = builder.into_inner().expect("finish proxy CA archive"); + let mut archive = tar::Archive::new(archive.as_slice()); + let mut entries = archive.entries().unwrap(); + let mut entry = entries.next().expect("proxy CA entry").unwrap(); + + assert_eq!( + entry.path().unwrap().as_ref(), + Path::new("upstream-proxy-ca-bundle.pem") + ); + assert_eq!(entry.header().uid().unwrap(), u64::from(SUPERVISOR_UID)); + assert_eq!(entry.header().gid().unwrap(), u64::from(SUPERVISOR_GID)); + assert_eq!(entry.header().mode().unwrap(), 0o644); + let mut actual = String::new(); + entry.read_to_string(&mut actual).unwrap(); + assert_eq!(actual, expected); + assert!(entries.next().is_none()); +} + +#[test] +fn sandbox_driver_config_cannot_override_proxy_ca_bundle() { + let config = runtime_config(); + let mut sandbox = test_sandbox(); + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "proxy_ca_bundle": "/workload/controlled-ca.pem" + }))); + + let error = DockerComputeDriver::validate_sandbox(&sandbox, &config) + .expect_err("sandbox driver config must not accept proxy_ca_bundle"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + assert!(error.message().contains("unknown field"), "{error}"); + assert!(error.message().contains("proxy_ca_bundle"), "{error}"); +} + fn test_workload_identity() -> ResolvedWorkloadIdentity { ResolvedWorkloadIdentity::new( 1234, diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index fc9d5aa8c2..3c8bb729a0 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -896,6 +896,13 @@ no_proxy = ".svc.cluster.local,10.0.0.0/8" # Optional root-owned host file containing user:pass. An http:// proxy also # requires proxy_auth_allow_insecure = true as an explicit acknowledgement. proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +proxy_auth_allow_insecure = false +# Last resort for proxy ACLs that require hostname CONNECT targets. The proxy +# then performs DNS resolution and becomes part of the effective egress boundary. +proxy_connect_by_hostname = false +# Operator-owned gateway-host PEM bundle trusted for an HTTPS proxy and for +# destination certificates re-signed by a TLS-intercepting proxy. +proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" # Project a host Unix Workload API socket into the supervisor for provider # token exchange. The socket parent must be a dedicated absolute directory. provider_spiffe_workload_api_socket = "/run/spire/agent.sock" @@ -904,6 +911,30 @@ provider_spiffe_workload_api_socket = "/run/spire/agent.sock" Use `sandbox_label` for Docker configurations. The legacy `sandbox_namespace` key is rejected. +Docker accepts `http://` and `https://` proxy URLs in explicit +`scheme://host:port` form. `no_proxy` bypasses only the corporate proxy; +OpenShell policy still applies. Proxy URLs cannot embed credentials. Supply a +`user:pass` file with `proxy_auth_file`, and set +`proxy_auth_allow_insecure = true` only to acknowledge that Basic credentials +travel in cleartext to an `http://` proxy. + +`proxy_ca_bundle` requires `https_proxy`, but the configured proxy URL may use +either scheme because a plain HTTP proxy can still intercept destination TLS. +The file must be a readable, bounded PEM bundle containing a usable certificate +authority. Docker validates it at gateway startup and again before each +supervisor launch. Missing, unreadable, empty, oversized, malformed, and +certificate-free bundles fail closed rather than falling back to default trust +or direct egress. + +The gateway-host path is operator-owned and cannot be selected through sandbox +environment, image contents, or `template.driver_config.docker`. The driver +copies the validated contents into its supervisor-only named volume and passes +the fixed path `/.openshell/supervisor/upstream-proxy-ca-bundle.pem` to the +companion supervisor. The bundle extends trust for the TLS connection to an +`https://` proxy, supervisor connections to re-signed upstream certificates, +and the combined trust bundle exposed to workload processes. The gateway-host +path is never mounted into or exposed to the workload container. + ### Podman Each Podman sandbox uses two containers. The workload container runs `openshell-sandbox` with `network=none`; the supervisor container runs on the host network and initiates policy-approved upstream connections. A private volume carries their authenticated Unix-domain socket. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 09a2771ff1..f52a571bd2 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -96,6 +96,26 @@ Common options in `[openshell.drivers.docker]` are `socket_path`, `grpc_endpoint Docker Desktop must have host networking enabled, and it cannot use Enhanced Container Isolation. Set `grpc_endpoint` when sandboxes cannot reach the gateway on host loopback. For GPU sandboxes, configure Docker CDI before starting the gateway. +### Docker Corporate Proxy Egress + +For proxy-required networks, the Docker driver accepts `https_proxy`, +`no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, +`proxy_connect_by_hostname`, and `proxy_ca_bundle`. The companion supervisor +chains policy-approved TLS tunnels through the proxy with HTTP CONNECT. + +`proxy_ca_bundle` is an operator-owned gateway-host PEM path. Docker validates +and copies the bundle into its supervisor-only named volume, so this also works +with remote Docker daemons and does not expose the host path to the workload. +The supervisor uses the bundle for an HTTPS proxy connection and for upstream +certificates re-signed by a TLS-intercepting proxy. Workload processes receive +the same corporate roots through their generated combined trust bundle. + +Sandbox environment, image contents, and `template.driver_config.docker` +cannot alter these settings. Invalid proxy relationships or CA files prevent +the gateway or sandbox from starting and never degrade to direct egress. See +the [Gateway Configuration File](/how-it-works/gateways/configuration) for URL, +authentication, `NO_PROXY`, hostname CONNECT, and CA validation details. + ### Docker Mounts Mount existing named volumes or `tmpfs` through driver config. Label volumes so resource admission accepts them: diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index b492c8ac86..4d15f09d20 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -68,6 +68,11 @@ name = "docker_preflight" path = "tests/docker_preflight.rs" required-features = ["e2e-docker"] +[[test]] +name = "docker_corporate_proxy" +path = "tests/docker_corporate_proxy.rs" +required-features = ["e2e-docker"] + [[test]] name = "driver_config_volume" path = "tests/driver_config_volume.rs" diff --git a/e2e/rust/tests/docker_corporate_proxy.rs b/e2e/rust/tests/docker_corporate_proxy.rs new file mode 100644 index 0000000000..85584b966f --- /dev/null +++ b/e2e/rust/tests/docker_corporate_proxy.rs @@ -0,0 +1,1180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-docker")] + +//! Cross-layer E2E coverage for corporate forward-proxy egress from Docker +//! sandboxes (issue #3545). +//! +//! The Docker counterpart of `podman_corporate_proxy.rs`. It drives the whole +//! chain end to end: +//! +//! gateway TOML → Docker driver config → companion supervisor archive → +//! supervisor CLI parsing → policy evaluation → proxied CONNECT +//! +//! and asserts the properties only a real run can establish: +//! +//! 1. A policy-approved HTTPS request reaches its destination *through* the +//! proxy, with a validated IP as the CONNECT target. +//! 2. A policy-denied destination never reaches the proxy at all. +//! 3. Credentials arrive through the supervisor-volume file — the proxy answers +//! 407 to an unauthenticated CONNECT, so a 200 proves delivery. +//! 4. A port-qualified `no_proxy` entry bypasses the proxy for that port only. +//! 5. An `https://` proxy works when its CA is supplied via `proxy_ca_bundle`. +//! 6. A TLS-intercepting proxy CA is trusted by both inspected supervisor +//! upstream TLS and a raw workload TLS connection. +//! 7. An incoherent setting is fatal at gateway startup rather than +//! degrading to a direct dial. +//! +//! Fixtures run as host processes and are reached by the host-networked +//! supervisor. `host.openshell.internal` is normalized to the gateway host +//! address, including the CI job-container address when Docker runs on the +//! host daemon. + +use std::fmt::Write as _; +use std::io::Write as _; +use std::path::PathBuf; +use std::time::Duration; + +use openshell_e2e::harness::cli::wait_for_healthy; +use openshell_e2e::harness::gateway::ManagedGateway; +use openshell_e2e::harness::host_process::HostPythonFixture; +use openshell_e2e::harness::port::find_free_port; +use openshell_e2e::harness::sandbox::SandboxGuard; +use serial_test::serial; +use tempfile::NamedTempFile; + +/// The OpenShell alias for the gateway host. +const HOST_ALIAS: &str = "host.openshell.internal"; +const PROXY_USER: &str = "proxyuser"; +const PROXY_PASS: &str = "proxypass"; + +const ALLOWED_MARKER: &str = "docker-corp-proxy-e2e-allowed-upstream"; +const DENIED_MARKER: &str = "docker-corp-proxy-e2e-denied-upstream"; +const BYPASS_MARKER: &str = "docker-corp-proxy-e2e-bypass-upstream"; +const READY_MARKER: &str = "docker-corp-proxy-e2e-workload-done"; + +/// Ports the fixtures bind and the workload addresses them by. +struct FixturePorts { + proxy: u16, + allowed: u16, + denied: u16, + bypass: u16, +} + +impl FixturePorts { + fn pick() -> Self { + Self { + proxy: find_free_port(), + allowed: find_free_port(), + denied: find_free_port(), + bypass: find_free_port(), + } + } +} + +/// Python helper retained by the shared proxy fixture shape. +const HOST_REWRITE: &str = " +def dial_host(host): + return host +"; + +/// A forward proxy that requires Basic auth and logs every CONNECT it sees. +/// +/// The log lines are the test's evidence: they record the exact CONNECT target +/// (proving validated-IP form, and which destination ports were proxied at +/// all) and whether credentials arrived. +fn proxy_script(port: u16) -> String { + format!( + r#" +import base64, select, socket, threading +{HOST_REWRITE} +EXPECTED = 'Basic ' + base64.b64encode(b'{PROXY_USER}:{PROXY_PASS}').decode() + +def log(msg): + print(msg, flush=True) + +def read_head(conn): + data = b'' + while b'\r\n\r\n' not in data: + chunk = conn.recv(4096) + if not chunk: + return None + data += chunk + if len(data) > 65536: + return None + return data + +def pipe(a, b): + try: + while True: + ready, _, _ = select.select([a, b], [], []) + for sock in ready: + chunk = sock.recv(65536) + if not chunk: + return + (b if sock is a else a).sendall(chunk) + except OSError: + return + +def handle(conn): + try: + head = read_head(conn) + if head is None: + # Readiness probes connect and close without sending a request. + return + lines = head.decode('latin-1').split('\r\n') + parts = lines[0].split() + if len(parts) < 2 or parts[0].upper() != 'CONNECT': + log('NON_CONNECT %s' % lines[0]) + conn.sendall(b'HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\n\r\n') + return + target = parts[1] + auth = None + for line in lines[1:]: + if line.lower().startswith('proxy-authorization:'): + auth = line.split(':', 1)[1].strip() + if auth != EXPECTED: + log('CONNECT %s auth=fail' % target) + conn.sendall(b'HTTP/1.1 407 Proxy Authentication Required\r\n' + b'Proxy-Authenticate: Basic realm="corp"\r\n' + b'Content-Length: 0\r\n\r\n') + return + host, _, port = target.rpartition(':') + host = host.strip('[]') + try: + upstream = socket.create_connection((dial_host(host), int(port)), timeout=10) + except OSError: + log('CONNECT %s auth=ok dial=fail' % target) + conn.sendall(b'HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n') + return + log('CONNECT %s auth=ok' % target) + conn.sendall(b'HTTP/1.1 200 Connection Established\r\n\r\n') + pipe(conn, upstream) + upstream.close() + except OSError: + pass + finally: + conn.close() + +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind(('0.0.0.0', {port})) +server.listen(64) +log('proxy-listening') +while True: + client, _ = server.accept() + threading.Thread(target=handle, args=(client,), daemon=True).start() +"# + ) +} + +// Delimits the proxy's CA certificate in its output, so the test can recover it +// and hand it back as the corporate CA bundle. +const CA_BEGIN: &str = "---PROXY-CA-BEGIN---"; +const CA_END: &str = "---PROXY-CA-END---"; + +/// A forward proxy that terminates TLS with a CA-signed certificate and logs +/// every CONNECT it sees. +/// +/// It mints a corporate CA and a listener leaf signed by it (SAN = the host +/// alias, which is the name the supervisor uses for SNI), serves the leaf, and +/// prints the CA between [`CA_BEGIN`]/[`CA_END`] so the test can trust it via +/// `proxy_ca_bundle`. The listener certificate must be a leaf: rustls rejects a +/// `CA:TRUE` certificate presented as an end-entity certificate. No Basic auth +/// here — this test isolates the `https://` proxy plus corporate-CA path. +/// Mint a corporate CA plus a listener leaf signed by it, and print the CA. +/// +/// Split out of [`tls_proxy_script`] to keep that function within clippy's +/// line budget. The listener certificate must be a leaf: rustls rejects a +/// `CA:TRUE` certificate presented as an end-entity certificate, so a single +/// self-signed `openssl req -x509` certificate cannot serve as both the +/// anchor and the listener identity. Signing a leaf also matches what a real +/// intercepting proxy does. +fn tls_proxy_pki_preamble() -> String { + format!( + r" +workdir = tempfile.mkdtemp() +ca_key = os.path.join(workdir, 'ca-key.pem') +ca_crt = os.path.join(workdir, 'ca.pem') +leaf_key = os.path.join(workdir, 'leaf-key.pem') +leaf_csr = os.path.join(workdir, 'leaf.csr') +leaf_crt = os.path.join(workdir, 'leaf.pem') +chain = os.path.join(workdir, 'chain.pem') +ext = os.path.join(workdir, 'leaf.ext') + +def run(*args): + subprocess.run(args, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + +run('openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', + '-keyout', ca_key, '-out', ca_crt, '-days', '1', + '-subj', '/CN=docker-corp-proxy-e2e-ca', + '-addext', 'basicConstraints=critical,CA:TRUE', + '-addext', 'keyUsage=critical,keyCertSign,cRLSign') + +with open(ext, 'w') as fh: + fh.write('basicConstraints=critical,CA:FALSE\n' + 'subjectAltName=DNS:{HOST_ALIAS}\n' + 'extendedKeyUsage=serverAuth\n' + 'keyUsage=critical,digitalSignature,keyEncipherment\n') +run('openssl', 'req', '-newkey', 'rsa:2048', '-nodes', + '-keyout', leaf_key, '-out', leaf_csr, '-subj', '/CN={HOST_ALIAS}') +run('openssl', 'x509', '-req', '-in', leaf_csr, '-CA', ca_crt, '-CAkey', ca_key, + '-CAcreateserial', '-out', leaf_crt, '-days', '1', '-extfile', ext) + +with open(chain, 'w') as out: + for part in (leaf_crt, ca_crt): + with open(part) as fh: + out.write(fh.read()) + +with open(ca_crt) as fh: + print('{CA_BEGIN}\n' + fh.read() + '{CA_END}', flush=True) +" + ) +} + +fn tls_proxy_script(port: u16) -> String { + let pki = tls_proxy_pki_preamble(); + format!( + r" +import os, select, socket, ssl, subprocess, tempfile, threading +{HOST_REWRITE} +{pki} + +def log(msg): + print(msg, flush=True) + +def read_head(conn): + data = b'' + while b'\r\n\r\n' not in data: + chunk = conn.recv(4096) + if not chunk: + return None + data += chunk + if len(data) > 65536: + return None + return data + +def pipe(a, b): + try: + while True: + ready, _, _ = select.select([a, b], [], []) + for sock in ready: + chunk = sock.recv(65536) + if not chunk: + return + (b if sock is a else a).sendall(chunk) + except OSError: + return + +def handle(conn): + try: + head = read_head(conn) + if head is None: + return + lines = head.decode('latin-1').split('\r\n') + parts = lines[0].split() + if len(parts) < 2 or parts[0].upper() != 'CONNECT': + log('NON_CONNECT %s' % lines[0]) + conn.sendall(b'HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\n\r\n') + return + target = parts[1] + host, _, port = target.rpartition(':') + host = host.strip('[]') + try: + upstream = socket.create_connection((dial_host(host), int(port)), timeout=10) + except OSError: + log('CONNECT %s dial=fail' % target) + conn.sendall(b'HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n') + return + log('CONNECT %s ok' % target) + conn.sendall(b'HTTP/1.1 200 Connection Established\r\n\r\n') + pipe(conn, upstream) + upstream.close() + except OSError: + pass + finally: + conn.close() + +ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +ctx.load_cert_chain(chain, leaf_key) +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind(('0.0.0.0', {port})) +server.listen(64) +log('tls-proxy-listening') +while True: + raw, _ = server.accept() + try: + conn = ctx.wrap_socket(raw, server_side=True) + except OSError: + # Readiness probes open a bare TCP connection; the failed handshake is + # expected and must not spam the log. + raw.close() + continue + threading.Thread(target=handle, args=(conn,), daemon=True).start() +" + ) +} + +/// A plain HTTP CONNECT proxy that intercepts tunneled destination TLS. +/// +/// The proxy presents a corporate-CA-signed certificate for +/// `host.openshell.internal`, then opens an unverified TLS connection to the +/// real fixture. This lets one test prove both consumers of the configured CA: +/// inspected traffic is verified by the supervisor, while `tls: skip` traffic +/// is verified directly by the workload through its combined trust bundle. +fn intercepting_proxy_script(port: u16) -> String { + let pki = tls_proxy_pki_preamble(); + format!( + r" +import select, socket, ssl, subprocess, tempfile, threading, os +{HOST_REWRITE} +{pki} + +def log(msg): + print(msg, flush=True) + +def read_head(conn): + data = b'' + while b'\r\n\r\n' not in data: + chunk = conn.recv(4096) + if not chunk: + return None + data += chunk + if len(data) > 65536: + return None + return data + +def relay_http(client, upstream): + # select() cannot safely drive SSLSocket application data: TLS control + # records can make the fd readable while recv() still waits for payload. + # This fixture serves one bodyless HTTP request per intercepted tunnel, so + # relay that exchange directly and let the upstream HTTP/1.0 close delimit + # the response. + request = read_head(client) + if request is None: + return + upstream.sendall(request) + while True: + chunk = upstream.recv(65536) + if not chunk: + return + client.sendall(chunk) + +server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +server_context.load_cert_chain(chain, leaf_key) +upstream_context = ssl._create_unverified_context() + +def handle(conn): + target = '' + try: + head = read_head(conn) + if head is None: + return + lines = head.decode('latin-1').split('\r\n') + parts = lines[0].split() + if len(parts) < 2 or parts[0].upper() != 'CONNECT': + conn.sendall(b'HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\n\r\n') + return + target = parts[1] + host, _, target_port = target.rpartition(':') + host = host.strip('[]') + try: + upstream_raw = socket.create_connection( + (dial_host(host), int(target_port)), timeout=10) + upstream = upstream_context.wrap_socket( + upstream_raw, server_hostname='{HOST_ALIAS}') + except OSError as err: + log('CONNECT %s mitm=dial-fail error=%r' % (target, err)) + conn.sendall(b'HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n') + return + conn.sendall(b'HTTP/1.1 200 Connection Established\r\n\r\n') + try: + client = server_context.wrap_socket(conn, server_side=True) + except OSError as err: + log('CONNECT %s mitm=client-tls-fail error=%r' % (target, err)) + return + log('CONNECT %s mitm=ok' % target) + relay_http(client, upstream) + upstream.close() + client.close() + except OSError as err: + log('CONNECT %s mitm=error error=%r' % (target, err)) + finally: + try: + conn.close() + except OSError: + pass + +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind(('0.0.0.0', {port})) +server.listen(64) +log('intercepting-proxy-listening') +while True: + conn, _ = server.accept() + threading.Thread(target=handle, args=(conn,), daemon=True).start() +" + ) +} + +/// Extract the proxy's CA certificate PEM from its output. +fn ca_cert_from_logs(logs: &str) -> Result { + let start = logs + .find(CA_BEGIN) + .ok_or_else(|| format!("proxy CA begin marker not found in logs:\n{logs}"))? + + CA_BEGIN.len(); + let end = logs[start..] + .find(CA_END) + .ok_or_else(|| format!("proxy CA end marker not found in logs:\n{logs}"))?; + Ok(logs[start..start + end].trim().to_string()) +} + +/// A TLS server with a self-signed certificate, serving one identifying marker. +/// +/// The workload uses an unverified TLS context, so the certificate only needs +/// to exist — but the handshake itself must be real, because it is what proves +/// bytes flowed end to end through the tunnel. +fn tls_upstream_script(marker: &str, port: u16) -> String { + format!( + r#" +import http.server, os, ssl, subprocess, tempfile + +workdir = tempfile.mkdtemp() +key = os.path.join(workdir, 'key.pem') +crt = os.path.join(workdir, 'cert.pem') +subprocess.run( + ['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', + '-keyout', key, '-out', crt, '-days', '1', '-subj', '/CN={HOST_ALIAS}'], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = b'{{"upstream":"{marker}"}}' + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + pass + +class Server(http.server.HTTPServer): + # Readiness probes open a bare TCP connection and close it; the failed + # handshake is expected and must not spam the log. + def handle_error(self, request, client_address): + pass + +ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +ctx.load_cert_chain(crt, key) +server = Server(('0.0.0.0', {port}), Handler) +server.socket = ctx.wrap_socket(server.socket, server_side=True) +print('tls-upstream-listening', flush=True) +server.serve_forever() +"# + ) +} + +/// Workload: one approved HTTPS request through the proxy, one policy-denied, +/// and one that `no_proxy` should send direct. +/// +fn workload_script(ports: &FixturePorts) -> String { + let (allowed, denied, bypass) = (ports.allowed, ports.denied, ports.bypass); + format!( + r" +import json, ssl, time, urllib.request + +ctx = ssl._create_unverified_context() + +def fetch(url, retries): + last = {{'status': -1, 'error': 'not attempted'}} + for attempt in range(retries): + try: + with urllib.request.urlopen(url, timeout=30, context=ctx) as resp: + return {{'status': resp.status, 'body': resp.read().decode()}} + except Exception as err: + last = {{'status': -1, 'error': str(err)}} + time.sleep(1) + return last + +# The approved requests are retried: policy reload during sandbox startup can +# transiently surface as a 403 in the forward proxy. +print('ALLOWED_RESULT ' + json.dumps( + fetch('https://{HOST_ALIAS}:{allowed}/', 6)), flush=True) +print('BYPASS_RESULT ' + json.dumps( + fetch('https://{HOST_ALIAS}:{bypass}/', 6)), flush=True) +# The denied request must fail, so a single attempt is enough. +print('DENIED_RESULT ' + json.dumps( + fetch('https://{HOST_ALIAS}:{denied}/', 1)), flush=True) +print('{READY_MARKER}', flush=True) +" + ) +} + +/// Workload variant that uses the system/default trust store. +/// +/// The inspected endpoint proves the supervisor accepts the corporate +/// re-signing CA for upstream TLS. The `tls: skip` endpoint proves the same CA +/// reached the workload trust bundle because Python verifies it directly. +fn trusted_workload_script(ports: &FixturePorts) -> String { + let (inspected, raw, denied) = (ports.allowed, ports.bypass, ports.denied); + format!( + r" +import json, ssl, time, urllib.request + +ctx = ssl.create_default_context() + +def fetch(url, retries): + last = {{'status': -1, 'error': 'not attempted'}} + for attempt in range(retries): + try: + with urllib.request.urlopen(url, timeout=30, context=ctx) as resp: + return {{'status': resp.status, 'body': resp.read().decode()}} + except Exception as err: + last = {{'status': -1, 'error': str(err)}} + time.sleep(1) + return last + +print('INSPECTED_RESULT ' + json.dumps( + fetch('https://{HOST_ALIAS}:{inspected}/', 6)), flush=True) +print('RAW_RESULT ' + json.dumps( + fetch('https://{HOST_ALIAS}:{raw}/', 6)), flush=True) +print('DENIED_RESULT ' + json.dumps( + fetch('https://{HOST_ALIAS}:{denied}/', 1)), flush=True) +print('{READY_MARKER}', flush=True) +" + ) +} + +/// Policy allowing the proxied and the bypassed upstream, but not the denied +/// one. `tls: skip` keeps the tunnel raw so the workload's TLS session runs end +/// to end, which is what makes the proxied CONNECT path observable. +fn policy_yaml(ports: &FixturePorts) -> String { + let (allowed, bypass) = (ports.allowed, ports.bypass); + format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /proc + - /dev/urandom + - /app + - /etc + - /var/log + read_write: + - /sandbox + - /tmp + - /dev/null + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + docker_corporate_proxy_e2e: + name: docker_corporate_proxy_e2e + endpoints: + - host: {HOST_ALIAS} + port: {allowed} + tls: skip + enforcement: enforce + allowed_ips: + - "10.0.0.0/8" + - "172.0.0.0/8" + - "192.168.0.0/16" + - "fc00::/7" + - host: {HOST_ALIAS} + port: {bypass} + tls: skip + enforcement: enforce + allowed_ips: + - "10.0.0.0/8" + - "172.0.0.0/8" + - "192.168.0.0/16" + - "fc00::/7" + binaries: + - path: /usr/bin/curl + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.venv/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +"# + ) +} + +/// Allow one auto-inspected TLS endpoint and one raw `tls: skip` endpoint. +fn interception_policy_yaml(ports: &FixturePorts) -> String { + let (inspected, raw) = (ports.allowed, ports.bypass); + format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + docker_corporate_proxy_interception_e2e: + name: docker_corporate_proxy_interception_e2e + endpoints: + - host: {HOST_ALIAS} + port: {inspected} + enforcement: enforce + allowed_ips: ["10.0.0.0/8", "172.0.0.0/8", "192.168.0.0/16", "fc00::/7"] + - host: {HOST_ALIAS} + port: {raw} + tls: skip + enforcement: enforce + allowed_ips: ["10.0.0.0/8", "172.0.0.0/8", "192.168.0.0/16", "fc00::/7"] + binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.venv/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +"# + ) +} + +/// Inserts corporate-proxy keys into the harness-generated Docker table and +/// restores the original file when dropped. +struct GatewayProxyConfig { + config_path: PathBuf, + original: Vec, + restored: bool, +} + +fn insert_docker_driver_settings(config: &str, extra: &str) -> Result { + let table_header = "[openshell.drivers.docker]"; + let table_start = config + .find(table_header) + .ok_or_else(|| format!("gateway config has no {table_header} table"))?; + let table_body = table_start + table_header.len(); + let insert_at = config[table_body..] + .find("\n[") + .map_or(config.len(), |offset| table_body + offset); + + let mut updated = String::with_capacity(config.len() + extra.len() + 2); + updated.push_str(&config[..insert_at]); + if !updated.ends_with('\n') { + updated.push('\n'); + } + updated.push_str(extra); + if !extra.ends_with('\n') { + updated.push('\n'); + } + updated.push_str(&config[insert_at..]); + Ok(updated) +} + +#[test] +fn proxy_settings_are_inserted_before_nested_driver_tables() { + let config = r#"[openshell.drivers.docker] +socket_path = "/var/run/docker.sock" + +[openshell.drivers.docker.resource_admission] +enabled = true +"#; + + let updated = insert_docker_driver_settings(config, "https_proxy = \"http://proxy\"\n") + .expect("insert proxy settings"); + let proxy_position = updated + .find("https_proxy = \"http://proxy\"") + .expect("proxy setting is present"); + let nested_table_position = updated + .find("[openshell.drivers.docker.resource_admission]") + .expect("nested table is preserved"); + + assert!(proxy_position < nested_table_position); + assert!(updated.contains("enabled = true")); +} + +impl GatewayProxyConfig { + /// Locate the gateway's `--config` path from the wrapper's args file. + fn config_path_from_args() -> Result { + let args_file = std::env::var("OPENSHELL_E2E_GATEWAY_ARGS_FILE") + .map_err(|_| "OPENSHELL_E2E_GATEWAY_ARGS_FILE must be set".to_string())?; + let raw = std::fs::read(&args_file) + .map_err(|err| format!("read gateway args file '{args_file}': {err}"))?; + let args: Vec = raw + .split(|byte| *byte == 0) + .filter(|arg| !arg.is_empty()) + .map(|arg| String::from_utf8_lossy(arg).into_owned()) + .collect(); + args.iter() + .position(|arg| arg == "--config") + .and_then(|index| args.get(index + 1)) + .map(PathBuf::from) + .ok_or_else(|| format!("no --config argument in gateway args file '{args_file}'")) + } + + /// Insert raw TOML lines into `[openshell.drivers.docker]` and restart the + /// gateway, without waiting for it to become healthy. + /// + /// Used directly by the fail-closed case, which expects the gateway *not* + /// to come up. + fn apply_raw(extra: &str) -> Result { + let config_path = Self::config_path_from_args()?; + let original = std::fs::read(&config_path) + .map_err(|err| format!("read gateway config '{}': {err}", config_path.display()))?; + + let config = std::str::from_utf8(&original).map_err(|err| { + format!( + "gateway config '{}' is not UTF-8: {err}", + config_path.display() + ) + })?; + let updated = insert_docker_driver_settings(config, extra) + .map_err(|err| format!("gateway config '{}': {err}", config_path.display()))?; + std::fs::write(&config_path, updated.as_bytes()) + .map_err(|err| format!("write gateway config '{}': {err}", config_path.display()))?; + + let guard = Self { + config_path, + original, + restored: false, + }; + let gateway = ManagedGateway::from_env()? + .ok_or_else(|| "managed gateway metadata disappeared".to_string())?; + gateway.stop()?; + gateway.start()?; + Ok(guard) + } + + /// Point the Docker driver at a corporate proxy and wait for the gateway to + /// come back healthy. + async fn apply( + proxy_url: &str, + auth_file: Option<&str>, + ca_bundle: Option<&str>, + no_proxy: Option<&str>, + ) -> Result { + let mut extra = format!("https_proxy = \"{proxy_url}\"\n"); + if let Some(auth_file) = auth_file { + let _ = write!( + extra, + "proxy_auth_file = \"{auth_file}\"\nproxy_auth_allow_insecure = true\n" + ); + } + if let Some(ca_bundle) = ca_bundle { + let _ = writeln!(extra, "proxy_ca_bundle = \"{ca_bundle}\""); + } + if let Some(no_proxy) = no_proxy { + let _ = writeln!(extra, "no_proxy = \"{no_proxy}\""); + } + let guard = Self::apply_raw(&extra)?; + wait_for_healthy(Duration::from_secs(120)).await?; + Ok(guard) + } + + /// Restore the original config and restart the gateway. + async fn restore(&mut self) -> Result<(), String> { + if self.restored { + return Ok(()); + } + std::fs::write(&self.config_path, &self.original).map_err(|err| { + format!( + "restore gateway config '{}': {err}", + self.config_path.display() + ) + })?; + let gateway = ManagedGateway::from_env()? + .ok_or_else(|| "managed gateway metadata disappeared".to_string())?; + gateway.stop()?; + gateway.start()?; + wait_for_healthy(Duration::from_secs(120)).await?; + self.restored = true; + Ok(()) + } +} + +impl Drop for GatewayProxyConfig { + fn drop(&mut self) { + if self.restored { + return; + } + // Panic path: put the original config back and synchronously restart + // the gateway so later test binaries in this run inherit neither the + // proxy settings on disk nor the configuration still loaded in the + // running process. + let _ = std::fs::write(&self.config_path, &self.original); + if let Ok(Some(gateway)) = ManagedGateway::from_env() { + let _ = gateway.stop(); + let _ = gateway.start(); + } + } +} + +/// Skip unless this run owns a Docker gateway it can reconfigure. +/// +/// The external-driver lane launches `openshell-driver-docker` from the shell +/// wrapper rather than from the gateway, so gateway config changes never reach +/// the driver and the test would assert against a driver that has no proxy +/// settings at all. +fn should_run(label: &str) -> bool { + if std::env::var("OPENSHELL_E2E_DRIVER").as_deref() != Ok("docker") { + eprintln!("Skipping {label}: e2e driver is not docker"); + return false; + } + if std::env::var("OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER").as_deref() == Ok("1") { + eprintln!("Skipping {label}: the external Docker driver is not configured by the gateway"); + return false; + } + match ManagedGateway::from_env() { + Ok(Some(_)) => true, + Ok(None) => { + eprintln!("Skipping {label}: e2e gateway is not managed by this test run"); + false + } + Err(err) => panic!("load managed e2e gateway metadata: {err}"), + } +} + +/// Write a temp file and return its path as an owned `String`. +fn temp_file_with(contents: &str, label: &str) -> (NamedTempFile, String) { + let mut file = NamedTempFile::new().unwrap_or_else(|err| panic!("create {label}: {err}")); + file.write_all(contents.as_bytes()) + .unwrap_or_else(|err| panic!("write {label}: {err}")); + file.flush() + .unwrap_or_else(|err| panic!("flush {label}: {err}")); + let path = file + .path() + .to_str() + .unwrap_or_else(|| panic!("{label} path should be utf-8")) + .to_string(); + (file, path) +} + +/// Assert the workload's results and the proxy's own record of what it saw. +fn assert_proxied_egress(output: &str, proxy_logs: &str, ports: &FixturePorts) { + assert!( + output.contains(READY_MARKER), + "workload did not finish; output:\n{output}" + ); + + // The approved request succeeded and its body came from the upstream, + // proving bytes traversed the tunnel rather than the proxy short-circuiting. + assert!( + output.contains("ALLOWED_RESULT") && output.contains(ALLOWED_MARKER), + "approved HTTPS request should have reached the upstream through the proxy:\n{output}" + ); + + // The CONNECT target is a validated resolved address, not the hostname, + // and it arrived authenticated -- which is only possible if the credential + // staged into the overlay reached the supervisor. + assert!( + proxy_logs.lines().any(|line| { + line.starts_with("CONNECT ") + && line.contains(&format!(":{} auth=ok", ports.allowed)) + && !line.contains(HOST_ALIAS) + }), + "proxy should have seen an authenticated validated-IP CONNECT to the approved upstream:\n{proxy_logs}" + ); + assert!( + !proxy_logs.contains("auth=fail"), + "the proxy credential should have been sent on the first CONNECT:\n{proxy_logs}" + ); + + // The denied destination failed inside the sandbox, and never reached the + // proxy: policy stops it before the upstream dial. + assert!( + !proxy_logs.contains(&format!(":{}", ports.denied)), + "policy-denied destination must never reach the proxy:\n{proxy_logs}" + ); + assert!( + !output.contains(DENIED_MARKER), + "policy-denied upstream body must never reach the workload:\n{output}" + ); + + // The no_proxy entry is port-qualified, so this destination was dialed + // directly while the approved one above still went through the proxy. + assert!( + output.contains("BYPASS_RESULT") && output.contains(BYPASS_MARKER), + "no_proxy destination should have been reached by a direct dial:\n{output}" + ); + assert!( + !proxy_logs.contains(&format!(":{}", ports.bypass)), + "no_proxy destination must never reach the proxy:\n{proxy_logs}" + ); +} + +#[tokio::test] +#[serial(docker_corporate_proxy)] +async fn docker_corporate_proxy_routes_approved_tls_egress() { + if !should_run("corporate proxy test") { + return; + } + + let ports = FixturePorts::pick(); + + // ── Host fixtures, reached by the host supervisor ── + let proxy = HostPythonFixture::start(&proxy_script(ports.proxy), ports.proxy) + .await + .expect("start fake corporate proxy"); + let _allowed = HostPythonFixture::start( + &tls_upstream_script(ALLOWED_MARKER, ports.allowed), + ports.allowed, + ) + .await + .expect("start approved TLS upstream"); + let _denied = HostPythonFixture::start( + &tls_upstream_script(DENIED_MARKER, ports.denied), + ports.denied, + ) + .await + .expect("start denied TLS upstream"); + let _bypass = HostPythonFixture::start( + &tls_upstream_script(BYPASS_MARKER, ports.bypass), + ports.bypass, + ) + .await + .expect("start no_proxy TLS upstream"); + + // ── Point the Docker driver at the corporate proxy ──────────────────── + let (_auth_file, auth_path) = + temp_file_with(&format!("{PROXY_USER}:{PROXY_PASS}\n"), "proxy auth file"); + + let mut gateway_config = GatewayProxyConfig::apply( + &format!("http://{HOST_ALIAS}:{}", ports.proxy), + Some(&auth_path), + None, + // Port-qualified, so only this destination port bypasses the proxy. + Some(&format!("{HOST_ALIAS}:{}", ports.bypass)), + ) + .await + .expect("apply corporate proxy gateway config"); + + // ── Run the workload ────────────────────────────────────────────── + let (_policy, policy_path) = temp_file_with(&policy_yaml(&ports), "policy file"); + let script = workload_script(&ports); + let mut sandbox = + SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("create Docker sandbox behind the corporate proxy"); + + assert_proxied_egress( + &sandbox.create_output, + &proxy.logs().expect("read fake proxy logs"), + &ports, + ); + + sandbox.cleanup().await; + + gateway_config + .restore() + .await + .expect("restore gateway config"); +} + +#[tokio::test] +#[serial(docker_corporate_proxy)] +async fn docker_corporate_proxy_trusts_ca_bundle_for_https_proxy() { + if !should_run("https corporate proxy test") { + return; + } + + let ports = FixturePorts::pick(); + + let proxy = HostPythonFixture::start(&tls_proxy_script(ports.proxy), ports.proxy) + .await + .expect("start fake https corporate proxy"); + let _allowed = HostPythonFixture::start( + &tls_upstream_script(ALLOWED_MARKER, ports.allowed), + ports.allowed, + ) + .await + .expect("start approved TLS upstream"); + let _denied = HostPythonFixture::start( + &tls_upstream_script(DENIED_MARKER, ports.denied), + ports.denied, + ) + .await + .expect("start denied TLS upstream"); + let _bypass = HostPythonFixture::start( + &tls_upstream_script(BYPASS_MARKER, ports.bypass), + ports.bypass, + ) + .await + .expect("start second approved TLS upstream"); + + // The proxy prints its CA on startup; the supervisor must trust it to + // complete the TLS handshake with the proxy at all. + let ca_pem = ca_cert_from_logs(&proxy.logs().expect("read https proxy logs")) + .expect("recover corporate CA from proxy output"); + let (_ca_file, ca_path) = temp_file_with(&ca_pem, "corporate CA bundle"); + + let mut gateway_config = GatewayProxyConfig::apply( + &format!("https://{HOST_ALIAS}:{}", ports.proxy), + None, + Some(&ca_path), + None, + ) + .await + .expect("apply https corporate proxy gateway config"); + + let (_policy, policy_path) = temp_file_with(&policy_yaml(&ports), "policy file"); + let script = workload_script(&ports); + let mut sandbox = + SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("create Docker sandbox behind the https corporate proxy"); + + let proxy_logs = proxy.logs().expect("read https proxy logs"); + assert!( + sandbox.create_output.contains(ALLOWED_MARKER), + "approved upstream body missing -- egress did not complete through the https proxy:\n{}", + sandbox.create_output + ); + assert!( + proxy_logs.lines().any(|line| { + line.starts_with("CONNECT ") + && line.contains(&format!(":{} ok", ports.allowed)) + && !line.contains(HOST_ALIAS) + }), + "https proxy should have seen a validated-IP CONNECT to the approved upstream:\n{proxy_logs}" + ); + assert!( + !proxy_logs.contains(&format!(":{}", ports.denied)), + "policy-denied destination must never reach the https proxy:\n{proxy_logs}" + ); + + sandbox.cleanup().await; + + gateway_config + .restore() + .await + .expect("restore gateway config"); +} + +#[tokio::test] +#[serial(docker_corporate_proxy)] +async fn docker_corporate_proxy_trusts_intercepted_destination_tls() { + if !should_run("TLS-intercepting corporate proxy test") { + return; + } + + let ports = FixturePorts::pick(); + let proxy = HostPythonFixture::start(&intercepting_proxy_script(ports.proxy), ports.proxy) + .await + .expect("start TLS-intercepting corporate proxy"); + let _inspected = HostPythonFixture::start( + &tls_upstream_script(ALLOWED_MARKER, ports.allowed), + ports.allowed, + ) + .await + .expect("start inspected TLS upstream"); + let _denied = HostPythonFixture::start( + &tls_upstream_script(DENIED_MARKER, ports.denied), + ports.denied, + ) + .await + .expect("start denied TLS upstream"); + let _raw = HostPythonFixture::start( + &tls_upstream_script(BYPASS_MARKER, ports.bypass), + ports.bypass, + ) + .await + .expect("start raw workload TLS upstream"); + + let ca_pem = ca_cert_from_logs(&proxy.logs().expect("read intercepting proxy logs")) + .expect("recover intercepting proxy CA"); + let (_ca_file, ca_path) = temp_file_with(&ca_pem, "intercepting proxy CA bundle"); + let mut gateway_config = GatewayProxyConfig::apply( + &format!("http://{HOST_ALIAS}:{}", ports.proxy), + None, + Some(&ca_path), + None, + ) + .await + .expect("apply TLS-intercepting proxy configuration"); + + let (_policy, policy_path) = temp_file_with( + &interception_policy_yaml(&ports), + "interception policy file", + ); + let script = trusted_workload_script(&ports); + let mut sandbox = + SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("create Docker sandbox behind TLS-intercepting proxy"); + + let output = &sandbox.create_output; + let proxy_logs = proxy.logs().expect("read TLS-intercepting proxy logs"); + assert!( + output.contains("INSPECTED_RESULT") && output.contains(ALLOWED_MARKER), + "supervisor-inspected TLS should trust the corporate re-signing CA:\n\ + workload output:\n{output}\nproxy output:\n{proxy_logs}" + ); + assert!( + output.contains("RAW_RESULT") && output.contains(BYPASS_MARKER), + "raw workload TLS should trust the corporate CA through the combined bundle:\n{output}" + ); + assert!( + proxy_logs.contains(&format!(":{} mitm=ok", ports.allowed)), + "intercepting proxy should see inspected upstream TLS:\n{proxy_logs}" + ); + assert!( + proxy_logs.contains(&format!(":{} mitm=ok", ports.bypass)), + "intercepting proxy should see raw workload TLS:\n{proxy_logs}" + ); + assert!( + !proxy_logs.contains(&format!(":{}", ports.denied)) && !output.contains(DENIED_MARKER), + "policy-denied destination must not reach the intercepting proxy:\n{proxy_logs}\n{output}" + ); + + sandbox.cleanup().await; + gateway_config + .restore() + .await + .expect("restore gateway config"); +} + +#[tokio::test] +#[serial(docker_corporate_proxy)] +async fn docker_corporate_proxy_rejects_incoherent_configuration() { + if !should_run("corporate proxy fail-closed test") { + return; + } + + // A bypass list without a proxy URL means the operator believed proxying + // was in effect. Accepting it would leave every dial direct while looking + // configured, so the gateway must refuse to start instead. + let mut gateway_config = GatewayProxyConfig::apply_raw("no_proxy = \"10.0.0.0/8\"\n") + .expect("apply incoherent corporate proxy gateway config"); + + let health = wait_for_healthy(Duration::from_secs(20)).await; + assert!( + health.is_err(), + "gateway must not serve traffic with an incoherent [openshell.drivers.docker] proxy table" + ); + + let log_path = std::env::var("OPENSHELL_E2E_GATEWAY_LOG").expect("gateway log path"); + let log = std::fs::read_to_string(&log_path).expect("read gateway log"); + // The message must name the offending key rather than surfacing as an + // opaque driver-readiness timeout. Matched loosely on the key names so + // this does not break on error-wrapping changes. + assert!( + log.contains("no_proxy") && log.contains("https_proxy"), + "the startup error must name the offending key; gateway log tail:\n{}", + log.lines().rev().take(40).collect::>().join("\n") + ); + + gateway_config + .restore() + .await + .expect("restore gateway config"); +} diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 2089f443b5..e7c44b08ad 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -296,6 +296,34 @@ Common findings: - The sandbox fails its enforcement probe: inspect the sandbox log for the exact nested seccomp user-notification, task-memory, Landlock, loopback DNS, or socket-injection check that failed. A runtime may return `ENOSYS` for `process_vm_readv` and `process_vm_writev` while satisfying the production parent-to-workload-child task-memory probe through `/proc//mem`; only failure of both backends is fatal. Do not add capabilities or switch to an unconfined seccomp profile; use a runtime whose default profile permits the unprivileged probe. - A GPU sandbox fails because Docker reports no discovered NVIDIA CDI devices: verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. Restart `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs, then restart or reload Docker and re-check `docker info`. +#### Corporate upstream proxy + +Docker corporate proxy settings are operator-owned fields under +`[openshell.drivers.docker]`. Confirm the complete proxy table and inspect the +companion supervisor command and logs: + +```bash +grep -A20 '^\[openshell.drivers.docker\]' | grep -E 'https_proxy|no_proxy|proxy_auth_file|proxy_auth_allow_insecure|proxy_connect_by_hostname|proxy_ca_bundle' +docker ps --filter label=openshell.ai/isolation-role=supervisor +docker inspect --format '{{json .Config.Cmd}} {{json .Mounts}}' +docker logs --tail=200 | grep -Ei 'upstream|connect|proxy|certificate' +``` + +`proxy_ca_bundle` names a gateway-host PEM file and requires `https_proxy`. +The proxy URL may use `http://` or `https://`; a plain HTTP proxy may still +re-sign destination TLS. Missing, unreadable, empty, oversized, malformed, or +certificate-free bundles fail closed. The Docker driver copies a validated +bundle into its supervisor-only named volume and passes the fixed path +`/.openshell/supervisor/upstream-proxy-ca-bundle.pem`. The gateway-host path +must not appear in container arguments, mounts, workload environment, or +`template.driver_config.docker`. + +An HTTPS proxy certificate error usually means the bundle lacks the proxy +listener issuer or its certificate does not match the proxy hostname. A +TLS-intercepted destination error means the re-signing issuer is missing or the +supervisor did not receive the bundle. Keep verification enabled and correct +the operator bundle. + During a graceful gateway restart, Docker, Podman, and VM sandboxes with running intent should stop before the gateway exits and restart after it returns. Check for `Stopped sandbox during gateway shutdown` and `Started diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index b85c9a6638..9974cb8a0d 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -214,6 +214,9 @@ fi if [[ -n "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE+x}" ]]; then + printf 'proxy_ca_bundle = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE}")" >>"${CONFIG_PATH}" +fi if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET+x}" ]]; then printf 'provider_spiffe_workload_api_socket = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET}")" >>"${CONFIG_PATH}" fi