From ca4573588dcb7b355a637291a4c99d92507f42d4 Mon Sep 17 00:00:00 2001 From: Artem Lytvyn Date: Tue, 22 Sep 2026 14:10:10 +0000 Subject: [PATCH 01/10] fix(driver-vm): resolve lifecycle requests on sandbox_id alone (#3305) Signed-off-by: Artem Lytvyn --- crates/openshell-driver-vm/src/driver.rs | 256 ++++++++++++++++++++--- 1 file changed, 231 insertions(+), 25 deletions(-) diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index fbd4230dec..68f1de47c8 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -576,6 +576,35 @@ struct SandboxRecord { deleting: bool, } +/// Resolve a lifecycle request to a registry key. +/// +/// A non-empty `sandbox_id` is authoritative: resolution uses that id alone and +/// never falls back to the name, so a request for an already-removed sandbox +/// reports absence instead of matching a same-named sandbox in another +/// workspace. Only a caller that supplies no id resolves by name, and because +/// sandbox names are unique per workspace rather than globally, a name matching +/// more than one record is rejected instead of decided by iteration order. +fn resolve_record_id( + registry: &HashMap, + sandbox_id: &str, + sandbox_name: &str, +) -> Result, Status> { + if !sandbox_id.is_empty() { + return Ok(registry.get_key_value(sandbox_id).map(|(id, _)| id.clone())); + } + + let mut matches = registry + .iter() + .filter(|(_, record)| record.snapshot.name == sandbox_name); + let first = matches.next().map(|(id, _)| id.clone()); + if matches.next().is_some() { + return Err(Status::failed_precondition(format!( + "sandbox_name {sandbox_name} matched more than one sandbox; supply sandbox_id" + ))); + } + Ok(first) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum OverlayPreparation { Fresh, @@ -1707,14 +1736,7 @@ impl VmDriver { } let record_id = { let registry = self.registry.lock().await; - if registry.contains_key(sandbox_id) { - Some(sandbox_id.to_string()) - } else { - registry - .iter() - .find(|(_, record)| record.snapshot.name == sandbox_name) - .map(|(id, _)| id.clone()) - } + resolve_record_id(®istry, sandbox_id, sandbox_name)? } .ok_or_else(|| Status::not_found("sandbox not found"))?; @@ -1799,16 +1821,13 @@ impl VmDriver { .map_err(|error| Status::invalid_argument(error.to_string()))?; let (record_id, state_dir, already_running) = { let registry = self.registry.lock().await; - let (id, record) = if let Some(entry) = registry.get_key_value(sandbox_id) { - entry - } else { - registry - .iter() - .find(|(_, record)| record.snapshot.name == sandbox_name) - .ok_or_else(|| Status::not_found("sandbox not found"))? - }; + let id = resolve_record_id(®istry, sandbox_id, sandbox_name)? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + let record = registry + .get(&id) + .ok_or_else(|| Status::not_found("sandbox not found"))?; ( - id.clone(), + id, record.state_dir.clone(), record.process.is_some() || record.provisioning_task.is_some(), ) @@ -1921,14 +1940,7 @@ impl VmDriver { let record_id = { let registry = self.registry.lock().await; - if let Some((id, _record)) = registry.get_key_value(sandbox_id) { - Some(id.clone()) - } else { - registry - .iter() - .find(|(_, record)| record.snapshot.name == sandbox_name) - .map(|(id, _)| id.clone()) - } + resolve_record_id(®istry, sandbox_id, sandbox_name)? }; let Some(record_id) = record_id else { @@ -10718,4 +10730,198 @@ mod tests { assert!(gpu.default_selection_supported); assert!(gpu.count_selection_supported); } + + /// Register a stopped, process-free record whose id, name, and workspace are + /// set independently. + /// + /// The other test helpers reuse one string for both id and name, so a test + /// built on them cannot express the cross-workspace name collision that + /// lifecycle resolution has to tolerate. + async fn insert_named_record( + driver: &VmDriver, + id: &str, + name: &str, + workspace: &str, + ) -> PathBuf { + let state_dir = sandbox_state_dir(&driver.config.state_dir, id).unwrap(); + create_private_dir_all(&state_dir).await.unwrap(); + driver.registry.lock().await.insert( + id.to_string(), + SandboxRecord { + snapshot: Sandbox { + id: id.to_string(), + name: name.to_string(), + workspace: workspace.to_string(), + ..Default::default() + }, + state_dir: state_dir.clone(), + process: None, + provisioning_task: None, + gpu_bdf: None, + deleting: false, + }, + ); + state_dir + } + + fn resolution_test_driver(state_dir: &Path) -> VmDriver { + let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + driver.config.state_dir = state_dir.to_path_buf(); + driver + } + + #[tokio::test] + async fn stop_targets_the_requested_id_when_two_workspaces_share_a_name() { + let temp = tempfile::tempdir().unwrap(); + let driver = resolution_test_driver(temp.path()); + let alpha = insert_named_record(&driver, "vm-alpha", "demo", "alpha").await; + let beta = insert_named_record(&driver, "vm-beta", "demo", "beta").await; + + driver + .stop_sandbox("vm-beta", "demo") + .await + .expect("stop by id should be accepted"); + + assert!( + beta.join(SANDBOX_STOPPED_FILE).exists(), + "the requested sandbox should have been stopped" + ); + assert!( + !alpha.join(SANDBOX_STOPPED_FILE).exists(), + "the same-named sandbox in another workspace must be untouched" + ); + } + + #[tokio::test] + async fn delete_targets_the_requested_id_when_two_workspaces_share_a_name() { + let temp = tempfile::tempdir().unwrap(); + let driver = resolution_test_driver(temp.path()); + let alpha = insert_named_record(&driver, "vm-alpha", "demo", "alpha").await; + insert_named_record(&driver, "vm-beta", "demo", "beta").await; + + let response = driver + .delete_sandbox("vm-beta", "demo") + .await + .expect("delete by id should be accepted"); + + assert!(response.deleted); + let registry = driver.registry.lock().await; + assert!(!registry.contains_key("vm-beta")); + assert!( + registry.contains_key("vm-alpha"), + "the same-named sandbox in another workspace must survive" + ); + assert!(alpha.exists(), "the surviving sandbox must keep its state"); + } + + #[tokio::test] + async fn a_repeated_delete_does_not_fall_back_to_a_same_named_sandbox() { + let temp = tempfile::tempdir().unwrap(); + let driver = resolution_test_driver(temp.path()); + let alpha = insert_named_record(&driver, "vm-alpha", "demo", "alpha").await; + insert_named_record(&driver, "vm-beta", "demo", "beta").await; + + let first = driver + .delete_sandbox("vm-beta", "demo") + .await + .expect("the first delete should be accepted"); + assert!(first.deleted); + + // Delete is idempotent, so a retry carrying the same id and name is + // ordinary caller behavior. The id is gone from the registry by now, and + // resolving it by name instead would destroy the sandbox in `alpha`. + let second = driver + .delete_sandbox("vm-beta", "demo") + .await + .expect("a repeated delete should be accepted"); + + assert!( + !second.deleted, + "a repeated delete must report that nothing was removed" + ); + assert!( + driver.registry.lock().await.contains_key("vm-alpha"), + "a repeated delete must not remove a same-named sandbox in another workspace" + ); + assert!(alpha.exists(), "the surviving sandbox must keep its state"); + } + + #[tokio::test] + async fn stop_and_start_reject_an_absent_id_that_shares_a_name() { + let temp = tempfile::tempdir().unwrap(); + let driver = resolution_test_driver(temp.path()); + let alpha = insert_named_record(&driver, "vm-alpha", "demo", "alpha").await; + + let stop_error = driver + .stop_sandbox("vm-absent", "demo") + .await + .expect_err("a supplied id that is not registered must not resolve by name"); + assert_eq!(stop_error.code(), Code::NotFound); + + let (start_authentication, _) = test_launch_authentication("absent"); + let start_error = driver + .start_sandbox( + "vm-absent", + "demo", + "g0000000000000001", + start_authentication, + ) + .await + .expect_err("a supplied id that is not registered must not resolve by name"); + assert_eq!(start_error.code(), Code::NotFound); + + assert!( + !alpha.join(SANDBOX_STOPPED_FILE).exists(), + "the same-named sandbox must not have been touched" + ); + } + + #[tokio::test] + async fn a_name_only_request_matching_two_sandboxes_is_rejected() { + let temp = tempfile::tempdir().unwrap(); + let driver = resolution_test_driver(temp.path()); + let alpha = insert_named_record(&driver, "vm-alpha", "demo", "alpha").await; + let beta = insert_named_record(&driver, "vm-beta", "demo", "beta").await; + + // Without an id the driver has nothing to disambiguate with: the request + // carries no workspace, and picking by iteration order would stop or + // delete an arbitrary one of the two. + for error in [ + driver.stop_sandbox("", "demo").await.unwrap_err(), + driver + .start_sandbox( + "", + "demo", + "g0000000000000001", + test_launch_authentication("ambiguous").0, + ) + .await + .unwrap_err(), + driver.delete_sandbox("", "demo").await.unwrap_err(), + ] { + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("matched more than one sandbox")); + } + + let registry = driver.registry.lock().await; + assert!(registry.contains_key("vm-alpha")); + assert!(registry.contains_key("vm-beta")); + assert!(!alpha.join(SANDBOX_STOPPED_FILE).exists()); + assert!(!beta.join(SANDBOX_STOPPED_FILE).exists()); + } + + #[tokio::test] + async fn a_name_only_request_still_resolves_a_unique_name() { + let temp = tempfile::tempdir().unwrap(); + let driver = resolution_test_driver(temp.path()); + let alpha = insert_named_record(&driver, "vm-alpha", "demo", "alpha").await; + insert_named_record(&driver, "vm-beta", "other", "beta").await; + + driver + .stop_sandbox("", "demo") + .await + .expect("an unambiguous name-only stop should still resolve"); + + assert!(alpha.join(SANDBOX_STOPPED_FILE).exists()); + } } From 7139df8ca53aa3a5210e40175264c8118d4c0e06 Mon Sep 17 00:00:00 2001 From: Varsha Date: Tue, 22 Sep 2026 14:11:06 +0000 Subject: [PATCH 02/10] fix(exec): preserve output after stdin EOF and verify stream completion (#3359) * fix(exec): preserve output after stdin EOF and verify stream completion Signed-off-by: Varsha Prasad Narsing * test(exec): cover fair duplex progress and live SDK completion Signed-off-by: Varsha Prasad Narsing * test(sdk): compare large exec buffers with native equality Signed-off-by: Varsha Prasad Narsing * test(exec): use workspace-scoped sandbox name in EOF regression Signed-off-by: Varsha Prasad Narsing --------- Signed-off-by: Varsha Prasad Narsing --- architecture/gateway.md | 20 + .../src/grpc/interactive_exec_tests.rs | 537 ++++++++++++++++++ crates/openshell-server/src/grpc/sandbox.rs | 183 ++++-- docs/sandboxes/manage-sandboxes.mdx | 6 + e2e/python/test_sandbox_api.py | 44 ++ sdk/go/README.md | 12 + sdk/go/docs/src/api/exec.md | 18 +- sdk/go/openshell/v1/exec.go | 30 + sdk/go/openshell/v1/exec_client.go | 108 ++-- sdk/go/openshell/v1/exec_client_test.go | 97 +++- sdk/go/openshell/v1/exec_control_test.go | 56 ++ sdk/typescript/README.md | 16 +- sdk/typescript/e2e/interactive-exec.mjs | 67 +++ sdk/typescript/src/client.test.ts | 210 ++++++- sdk/typescript/src/client.ts | 181 +++++- sdk/typescript/src/index.ts | 1 + tasks/typescript.toml | 5 + 17 files changed, 1442 insertions(+), 149 deletions(-) create mode 100644 crates/openshell-server/src/grpc/interactive_exec_tests.rs create mode 100644 sdk/go/openshell/v1/exec_control_test.go create mode 100644 sdk/typescript/e2e/interactive-exec.mjs diff --git a/architecture/gateway.md b/architecture/gateway.md index 0c221fb3a7..742fa5e189 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -959,6 +959,26 @@ alone — instead an unanswered keepalive on a wedged or orphaned relay closes t channel and returns the exec with an error. Once a command reports its exit status, the gateway also bounds how long it waits for the trailing channel close. +Interactive exec treats normal request-stream EOF as the end of stdin and resize +input. The gateway sends SSH EOF while keeping the output channel open until +command completion. Input errors terminate the operation rather than masquerading +as normal EOF. The input and output pumps are owned by the exec operation, so +timeout or response abandonment cannot leave a detached stdin task behind. +The pumps share polling fairly, and request processing yields cooperatively even +for ignored resize messages, so sustained input cannot monopolize the operation. + +Go and TypeScript interactive-exec helpers distinguish process exit from stream +completion. They consume the final gRPC status before reporting success and retain +an observed exit code if transport completion fails. Callers must drain output +concurrently with waiting for completion. + +TypeScript starts interactive exec eagerly and uses a bounded output queue between +the background receiver and the consumer. Cancellation wakes a receiver blocked +on that queue. Go exposes input closure through an optional session capability, +preserving the original interface for existing implementations. TypeScript also +preserves its original session interface; SDK-created sessions expose lifecycle +controls through an extended interface. + `ForwardTcp` is the client-facing byte stream for SSH and service forwarding. The first frame is a `TcpForwardInit` that carries the workspace-scoped sandbox name, an authorization token from `CreateSshSession`, and an explicit target: diff --git a/crates/openshell-server/src/grpc/interactive_exec_tests.rs b/crates/openshell-server/src/grpc/interactive_exec_tests.rs new file mode 100644 index 0000000000..63be73ff59 --- /dev/null +++ b/crates/openshell-server/src/grpc/interactive_exec_tests.rs @@ -0,0 +1,537 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Exercise the gateway's actual SSH input/output bridge with a controllable peer. + +use super::*; +use openshell_core::proto::{exec_sandbox_event, exec_sandbox_input}; +use russh::server::{Auth, ChannelOpenHandle, Handler, Msg, Session}; +use std::time::Duration; + +struct ExecPeer { + channel: Option>, + echo_tx: Option>>, + input: Arc>>, + eof: Arc, + release_output: Arc, + output_task: Option>, +} + +impl Drop for ExecPeer { + fn drop(&mut self) { + if let Some(task) = self.output_task.take() { + task.abort(); + } + } +} + +impl Handler for ExecPeer { + type Error = russh::Error; + + async fn auth_none(&mut self, _user: &str) -> Result { + Ok(Auth::Accept) + } + + async fn channel_open_session( + &mut self, + channel: russh::Channel, + reply: ChannelOpenHandle, + _session: &mut Session, + ) -> Result<(), Self::Error> { + reply.accept().await; + self.channel = Some(channel); + Ok(()) + } + + async fn exec_request( + &mut self, + channel: russh::ChannelId, + data: &[u8], + session: &mut Session, + ) -> Result<(), Self::Error> { + session.channel_success(channel)?; + if data == b"duplex" { + let channel = self.channel.take().unwrap(); + let release = self.release_output.clone(); + let (echo_tx, mut echo_rx) = mpsc::unbounded_channel::>(); + self.echo_tx = Some(echo_tx); + self.output_task = Some(tokio::spawn(async move { + let (reader, writer) = channel.split(); + // The handler receives stdin independently of output flow + // control, as a real process with separate I/O pumps would. + drop(reader); + while let Some(data) = echo_rx.recv().await { + writer.data_bytes(data.clone()).await.unwrap(); + writer.extended_data_bytes(1, data).await.unwrap(); + } + release.notified().await; + writer.exit_status(7).await.unwrap(); + writer.close().await.unwrap(); + })); + } else { + self.channel.take(); + } + if data == b"early" { + session.exit_status_request(channel, 0)?; + session.close(channel)?; + return Ok(()); + } + // Signal that SSH setup is complete without depending on input EOF. + session.data(channel, b"ready".to_vec())?; + Ok(()) + } + + async fn data( + &mut self, + _channel: russh::ChannelId, + data: &[u8], + _session: &mut Session, + ) -> Result<(), Self::Error> { + self.input.lock().unwrap().extend_from_slice(data); + if let Some(tx) = &self.echo_tx { + tx.send(data.to_vec()).unwrap(); + } + Ok(()) + } + + async fn channel_eof( + &mut self, + channel: russh::ChannelId, + session: &mut Session, + ) -> Result<(), Self::Error> { + self.eof.store(true, Ordering::SeqCst); + self.echo_tx.take(); + if self.output_task.is_some() { + return Ok(()); + } + let release = self.release_output.clone(); + let handle = session.handle(); + self.output_task = Some(tokio::spawn(async move { + release.notified().await; + if handle.data(channel, b"after eof".to_vec()).await.is_err() { + return; + } + let _ = handle + .extended_data(channel, 1, b"stderr after eof".to_vec()) + .await; + let _ = handle.exit_status_request(channel, 7).await; + let _ = handle.eof(channel).await; + let _ = handle.close(channel).await; + })); + Ok(()) + } + + async fn channel_close( + &mut self, + _channel: russh::ChannelId, + _session: &mut Session, + ) -> Result<(), Self::Error> { + // A client CLOSE before release must really prevent later output. + if let Some(task) = self.output_task.take() { + task.abort(); + } + Ok(()) + } +} + +struct Fixture { + port: u16, + input: Arc>>, + eof: Arc, + release_output: Arc, + server: tokio::task::JoinHandle<()>, +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.server.abort(); + } +} + +impl Fixture { + async fn new() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let mut config = russh::server::Config::default(); + config + .keys + .push(russh::keys::ssh_key::private::Ed25519Keypair::from_seed(&rand::random()).into()); + let input = Arc::new(std::sync::Mutex::new(Vec::new())); + let eof = Arc::new(AtomicBool::new(false)); + let release_output = Arc::new(tokio::sync::Notify::new()); + let handler = ExecPeer { + channel: None, + echo_tx: None, + input: input.clone(), + eof: eof.clone(), + release_output: release_output.clone(), + output_task: None, + }; + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + set_tcp_nodelay_best_effort(&socket); + if let Ok(session) = russh::server::run_stream(Arc::new(config), socket, handler).await + { + let _ = session.await; + } + }); + Self { + port, + input, + eof, + release_output, + server, + } + } +} + +async fn ready(rx: &mut mpsc::Receiver>) { + let event = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!( + matches!(event.payload, Some(exec_sandbox_event::Payload::Stdout(s)) if s.data == b"ready") + ); +} + +#[tokio::test] +async fn interactive_exec_drains_stdout_and_stderr_after_input_eof() { + let fixture = Fixture::new().await; + let (input_tx, input_rx) = mpsc::channel(2); + let (output_tx, mut output_rx) = mpsc::channel(2); + let exec = run_interactive_exec_with_russh( + fixture.port, + "test", + ReceiverStream::new(input_rx), + false, + false, + 0, + 0, + output_tx, + ); + let exercise = async { + ready(&mut output_rx).await; + input_tx + .send(Ok(ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Stdin(b"input".to_vec())), + })) + .await + .unwrap(); + drop(input_tx); + while !fixture.eof.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + // Keep the peer silent briefly after EOF so a premature SSH CLOSE is + // processed before output is released. + tokio::time::sleep(Duration::from_millis(50)).await; + fixture.release_output.notify_one(); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + while let Some(event) = output_rx.recv().await { + match event.unwrap().payload.unwrap() { + exec_sandbox_event::Payload::Stdout(s) => stdout.extend(s.data), + exec_sandbox_event::Payload::Stderr(s) => stderr.extend(s.data), + other @ exec_sandbox_event::Payload::Exit(_) => { + panic!("unexpected event: {other:?}") + } + } + } + assert_eq!(stdout, b"after eof"); + assert_eq!(stderr, b"stderr after eof"); + assert_eq!(*fixture.input.lock().unwrap(), b"input"); + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(exec, exercise) + }) + .await + .unwrap(); + assert_eq!(result.unwrap(), 7); +} + +#[tokio::test] +async fn interactive_exec_makes_progress_in_both_directions_before_eof() { + const CHUNKS: usize = 128; + const CHUNK_SIZE: usize = 64 * 1024; + const BATCH: usize = 4; + let fixture = Fixture::new().await; + let (input_tx, input_rx) = mpsc::channel(16); + let (output_tx, mut output_rx) = mpsc::channel(2); + let drained = tokio::sync::Notify::new(); + let progress = std::cell::Cell::new((0, 0)); + let exec = run_interactive_exec_with_russh( + fixture.port, + "duplex", + ReceiverStream::new(input_rx), + false, + false, + 0, + 0, + output_tx, + ); + let writer = async { + for chunk in 1..=CHUNKS { + input_tx + .send(Ok(ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Stdin(vec![b'x'; CHUNK_SIZE])), + })) + .await + .unwrap(); + if chunk % BATCH == 0 { + drained.notified().await; + } + } + // Keep the request stream open until BOTH output streams have drained. + // Bound in-flight data to exercise sustained interactive traffic without + // saturating both ends of the fixture's SSH transport simultaneously. + drop(input_tx); + }; + let reader = async { + ready(&mut output_rx).await; + let mut stdout = 0; + let mut stderr = 0; + let mut acknowledged = 0; + while stdout < CHUNKS * CHUNK_SIZE || stderr < CHUNKS * CHUNK_SIZE { + let bytes = match output_rx.recv().await.unwrap().unwrap().payload.unwrap() { + exec_sandbox_event::Payload::Stdout(s) => { + stdout += s.data.len(); + s.data + } + exec_sandbox_event::Payload::Stderr(s) => { + stderr += s.data.len(); + s.data + } + event @ exec_sandbox_event::Payload::Exit(_) => { + panic!("unexpected event: {event:?}") + } + }; + assert!(bytes.iter().all(|b| *b == b'x')); + progress.set((stdout, stderr)); + assert!(!fixture.eof.load(Ordering::SeqCst)); + if stdout.min(stderr) >= acknowledged + BATCH * CHUNK_SIZE { + acknowledged += BATCH * CHUNK_SIZE; + drained.notify_one(); + } + } + assert_eq!(stdout, CHUNKS * CHUNK_SIZE); + assert_eq!(stderr, CHUNKS * CHUNK_SIZE); + fixture.release_output.notify_one(); + while output_rx.recv().await.is_some() {} + }; + let (result, (), ()) = tokio::time::timeout(Duration::from_secs(30), async { + tokio::join!(exec, writer, reader) + }) + .await + .unwrap_or_else(|_| { + panic!( + "duplex stalled: input={}, output={:?}, eof={}", + fixture.input.lock().unwrap().len(), + progress.get(), + fixture.eof.load(Ordering::SeqCst) + ) + }); + assert_eq!(result.unwrap(), 7); + assert_eq!(fixture.input.lock().unwrap().len(), CHUNKS * CHUNK_SIZE); +} + +#[tokio::test] +async fn interactive_exec_ready_resize_stream_does_not_starve_output() { + use futures::StreamExt; + use std::sync::atomic::AtomicUsize; + + // Finite to make a regression fail rather than wedge the runtime forever. + // These frames have no SSH write await because this session has no PTY. + const FRAMES: usize = 100_000; + let fixture = Fixture::new().await; + let consumed = AtomicUsize::new(0); + let input = futures::stream::repeat_with(|| { + consumed.fetch_add(1, Ordering::SeqCst); + Ok(ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Resize( + openshell_core::proto::ExecSandboxWindowResize::default(), + )), + }) + }) + .take(FRAMES); + let (output_tx, mut output_rx) = mpsc::channel(2); + let exec = + run_interactive_exec_with_russh(fixture.port, "test", input, false, false, 0, 0, output_tx); + let reader = async { + ready(&mut output_rx).await; + assert!( + consumed.load(Ordering::SeqCst) < FRAMES, + "output must be delivered before the continuously ready input ends" + ); + drop(output_rx); + }; + let (result, ()) = + tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(exec, reader) }) + .await + .unwrap(); + assert_eq!(result.unwrap_err().code(), tonic::Code::Cancelled); +} + +#[tokio::test] +async fn interactive_exec_input_error_is_not_graceful_eof() { + for message in [ + Err(Status::cancelled("input cancelled")), + Ok(ExecSandboxInput { payload: None }), + Ok(ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Start( + ExecSandboxRequest::default(), + )), + }), + ] { + let expected = message + .as_ref() + .err() + .map_or(tonic::Code::InvalidArgument, Status::code); + let fixture = Fixture::new().await; + let (input_tx, input_rx) = mpsc::channel(1); + let (output_tx, mut output_rx) = mpsc::channel(1); + let exec = run_interactive_exec_with_russh( + fixture.port, + "test", + ReceiverStream::new(input_rx), + false, + false, + 0, + 0, + output_tx, + ); + let exercise = async { + ready(&mut output_rx).await; + input_tx.send(message).await.unwrap(); + while output_rx.recv().await.is_some() {} + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(exec, exercise) + }) + .await + .unwrap(); + assert_eq!(result.unwrap_err().code(), expected); + assert!(!fixture.eof.load(Ordering::SeqCst)); + } +} + +#[tokio::test] +async fn interactive_exec_response_drop_cancels_idle_input() { + let fixture = Fixture::new().await; + let (input_tx, input_rx) = mpsc::channel(1); + let (output_tx, mut output_rx) = mpsc::channel(1); + let exec = run_interactive_exec_with_russh( + fixture.port, + "test", + ReceiverStream::new(input_rx), + false, + false, + 0, + 0, + output_tx, + ); + let exercise = async { + ready(&mut output_rx).await; + drop(output_rx); + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(exec, exercise) + }) + .await + .unwrap(); + assert_eq!(result.unwrap_err().code(), tonic::Code::Cancelled); + assert!(input_tx.is_closed(), "stdin receiver must not outlive exec"); + assert!(!fixture.eof.load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn interactive_exec_parent_abort_drops_input_receiver() { + let fixture = Fixture::new().await; + let (input_tx, input_rx) = mpsc::channel(1); + let (output_tx, mut output_rx) = mpsc::channel(1); + let exec = tokio::spawn(run_interactive_exec_with_russh( + fixture.port, + "test", + ReceiverStream::new(input_rx), + false, + false, + 0, + 0, + output_tx, + )); + ready(&mut output_rx).await; + // An operation timeout drops the same future as this task abort. + exec.abort(); + assert!(exec.await.unwrap_err().is_cancelled()); + assert!( + input_tx.is_closed(), + "aborted exec must not leave a stdin task alive" + ); +} + +#[tokio::test] +async fn interactive_exec_response_drop_unblocks_full_output_queue() { + let fixture = Fixture::new().await; + let (input_tx, input_rx) = mpsc::channel(1); + let (output_tx, mut output_rx) = mpsc::channel(1); + // Use an empty request stream: EOF releases the peer's output once notified. + drop(input_tx); + let exec = run_interactive_exec_with_russh( + fixture.port, + "test", + ReceiverStream::new(input_rx), + false, + false, + 0, + 0, + output_tx, + ); + let exercise = async { + ready(&mut output_rx).await; + fixture.release_output.notify_one(); + while output_rx.is_empty() { + tokio::task::yield_now().await; + } + // stdout fills the one-slot queue while stderr still needs delivery. + drop(output_rx); + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(exec, exercise) + }) + .await + .unwrap(); + assert_eq!(result.unwrap_err().code(), tonic::Code::Cancelled); +} + +#[tokio::test] +async fn interactive_exec_early_exit_does_not_wait_for_stdin() { + let fixture = Fixture::new().await; + let (input_tx, input_rx) = mpsc::channel(1); + input_tx + .send(Ok(ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Stdin(vec![ + b'x'; + 4 * 1024 * 1024 + ])), + })) + .await + .unwrap(); + let (output_tx, _output_rx) = mpsc::channel(1); + let result = tokio::time::timeout( + Duration::from_secs(5), + run_interactive_exec_with_russh( + fixture.port, + "early", + ReceiverStream::new(input_rx), + false, + false, + 0, + 0, + output_tx, + ), + ) + .await + .unwrap(); + assert_eq!(result.unwrap(), 0); + assert!(input_tx.is_closed()); +} diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 6170201db9..b75f464f0b 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -77,6 +77,10 @@ const NO_LOGIN_SHELL_ENV: (&str, &str) = ("OPENSHELL_NO_LOGIN_SHELL", "1"); const MAX_TEMPLATES_PER_WORKSPACE: u32 = 1000; const MAX_CREATE_SERVICE_EXPOSURES: usize = 32; +#[cfg(test)] +#[path = "interactive_exec_tests.rs"] +mod interactive_exec_tests; + #[derive(Debug)] pub struct WatchSandboxStream { receiver: ReceiverStream>, @@ -2959,7 +2963,7 @@ async fn stream_interactive_exec_over_relay( )), })) .await; - let _ = proxy_task.await; + finish_interactive_exec_proxy(proxy_task).await; return Ok(()); } } else { @@ -2969,12 +2973,12 @@ async fn stream_interactive_exec_over_relay( let exit_code = match exec_result { Ok(code) => code, Err(status) => { - let _ = proxy_task.await; + finish_interactive_exec_proxy(proxy_task).await; return Err(status); } }; - let _ = proxy_task.await; + finish_interactive_exec_proxy(proxy_task).await; let _ = tx .send(Ok(ExecSandboxEvent { @@ -2987,17 +2991,28 @@ async fn stream_interactive_exec_over_relay( Ok(()) } +async fn finish_interactive_exec_proxy(mut task: tokio::task::JoinHandle<()>) { + if tokio::time::timeout(EXEC_POST_EXIT_CLOSE_TIMEOUT, &mut task) + .await + .is_err() + { + task.abort(); + let _ = task.await; + } +} + #[allow(clippy::too_many_arguments)] async fn run_interactive_exec_with_russh( local_proxy_port: u16, command: &str, - mut input_stream: tonic::Streaming, + mut input_stream: impl futures::Stream> + Unpin, request_tty: bool, no_login_shell: bool, cols: u32, rows: u32, tx: mpsc::Sender>, ) -> Result { + use futures::StreamExt; use openshell_core::proto::exec_sandbox_input::Payload; use russh::ChannelMsg; @@ -3073,83 +3088,125 @@ async fn run_interactive_exec_with_russh( let (mut read_half, write_half) = channel.split(); - let stdin_task = tokio::spawn(async move { - while let Ok(Some(msg)) = input_stream.message().await { + // Keep both directions independently polled, but owned by this operation. + // A detached stdin task would survive operation timeout and retain the SSH + // channel. Normal request EOF closes stdin only, never the output channel. + let input = async { + while let Some(msg) = input_stream.next().await { + // Even ignored non-PTY resize frames must cooperate with output + // and cancellation when the request stream stays continuously ready. + tokio::task::consume_budget().await; + let msg = msg?; match msg.payload { Some(Payload::Stdin(data)) => { - if write_half.data(std::io::Cursor::new(data)).await.is_err() { - break; - } + write_half + .data(std::io::Cursor::new(data)) + .await + .map_err(|_| { + Status::unavailable("exec relay failed while writing stdin") + })?; } Some(Payload::Resize(resize)) => { if request_tty { - let _ = write_half + write_half .window_change(resize.cols, resize.rows, 0, 0) - .await; + .await + .map_err(|_| Status::unavailable("exec relay failed while resizing"))?; } } - Some(Payload::Start(_)) | None => {} + Some(Payload::Start(_)) | None => { + return Err(Status::invalid_argument( + "expected stdin or resize after exec start", + )); + } } } - let _ = write_half.eof().await; - let _ = write_half.close().await; - }); + write_half + .eof() + .await + .map_err(|_| Status::unavailable("exec relay failed while closing stdin")) + }; - let mut exit_code: Option = None; - loop { - // Bound the post-ExitStatus wait against a lost Close. - let msg = if exit_code.is_some() { - match tokio::time::timeout(EXEC_POST_EXIT_CLOSE_TIMEOUT, read_half.wait()).await { - Ok(Some(msg)) => msg, - Ok(None) | Err(_) => break, - } - } else { - match read_half.wait().await { - Some(msg) => msg, - None => break, - } - }; - match msg { - ChannelMsg::Data { data } => { - let event = Ok(ExecSandboxEvent { - payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Stdout( - ExecSandboxStdout { - data: data.to_vec(), - }, - )), - }); - if tx.send(event).await.is_err() { - break; + let output = async { + let mut exit_code: Option = None; + loop { + // Bound the post-ExitStatus wait against a lost Close. + let msg = if exit_code.is_some() { + match tokio::time::timeout(EXEC_POST_EXIT_CLOSE_TIMEOUT, read_half.wait()).await { + Ok(Some(msg)) => msg, + Ok(None) | Err(_) => break, } - } - ChannelMsg::ExtendedData { data, .. } => { - let event = Ok(ExecSandboxEvent { - payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Stderr( - ExecSandboxStderr { - data: data.to_vec(), - }, - )), - }); - if tx.send(event).await.is_err() { - break; + } else { + match read_half.wait().await { + Some(msg) => msg, + None => break, } + }; + match msg { + ChannelMsg::Data { data } => { + let event = Ok(ExecSandboxEvent { + payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Stdout( + ExecSandboxStdout { + data: data.to_vec(), + }, + )), + }); + if tx.send(event).await.is_err() { + break; + } + } + ChannelMsg::ExtendedData { data, .. } => { + let event = Ok(ExecSandboxEvent { + payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Stderr( + ExecSandboxStderr { + data: data.to_vec(), + }, + )), + }); + if tx.send(event).await.is_err() { + break; + } + } + ChannelMsg::ExitStatus { exit_status } => { + let converted = i32::try_from(exit_status).unwrap_or(i32::MAX); + exit_code = Some(converted); + } + ChannelMsg::Close => break, + _ => {} } - ChannelMsg::ExitStatus { exit_status } => { - let converted = i32::try_from(exit_status).unwrap_or(i32::MAX); - exit_code = Some(converted); - } - ChannelMsg::Close => break, - _ => {} } - } - stdin_task.abort(); + exec_loop_result(exit_code) + }; - let _ = client - .disconnect(russh::Disconnect::ByApplication, "exec complete", "en") - .await; + let result = { + tokio::pin!(input, output); + let exchange = async { + tokio::select! { + result = &mut input => { + result?; + output.await + } + result = &mut output => result, + } + }; + tokio::select! { + biased; + () = tx.closed() => Err(Status::cancelled("exec response stream closed")), + result = exchange => result, + } + }; - exec_loop_result(exit_code) + // EOF above deliberately leaves this channel open until output completes. + // Bound cleanup even if the SSH peer is no longer making progress. + let _ = tokio::time::timeout(EXEC_POST_EXIT_CLOSE_TIMEOUT, write_half.close()).await; + let _ = tokio::time::timeout( + EXEC_POST_EXIT_CLOSE_TIMEOUT, + client.disconnect(russh::Disconnect::ByApplication, "exec complete", "en"), + ) + .await; + + result } /// Create a localhost SSH proxy that bridges to a relay `DuplexStream`. diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index f5ccbe3325..3cf3fdf838 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -329,6 +329,12 @@ OpenShell-managed SSH include file instead of cluttering your main ## Execute a Command in a Sandbox +For raw `ExecSandboxInteractive` clients, ending the request stream closes stdin +and terminal resize input. Continue reading the response to drain stdout/stderr +and receive the exit event and final gRPC status. Input EOF does not immediately +close the SSH output channel and is distinct from cancelling the RPC. With a PTY, +input closure is not equivalent to sending a terminal Ctrl-D keystroke. + Run a one-shot command inside a running sandbox without opening an interactive shell: ```shell diff --git a/e2e/python/test_sandbox_api.py b/e2e/python/test_sandbox_api.py index 6b7ff14dde..5885f376b2 100644 --- a/e2e/python/test_sandbox_api.py +++ b/e2e/python/test_sandbox_api.py @@ -200,6 +200,50 @@ def requests(): assert b"TTT" in stdout + stderr +def test_interactive_exec_drains_output_after_request_eof( + sandbox: Callable[..., Sandbox], + sandbox_client: SandboxClient, +) -> None: + with sandbox(delete_on_exit=True) as sb: + + def requests(): + yield openshell_pb2.ExecSandboxInput( + start=openshell_pb2.ExecSandboxRequest( + sandbox=sb.sandbox.name, + workspace_scope=datamodel_pb2.WorkspaceSelector(workspace="default"), + command=[ + "/bin/sh", + "-c", + "input=$(cat); sleep 0.2; " + "printf '%s' \"$input\"; printf 'drained-stderr' >&2; exit 7", + ], + tty=False, + execution_timeout=duration_pb2.Duration(seconds=20), + ) + ) + yield openshell_pb2.ExecSandboxInput(stdin=b"drained-stdout") + # End requests before the command emits output. The receive + # direction must survive long enough to drain both output streams. + + stdout = bytearray() + stderr = bytearray() + exit_codes: list[int] = [] + for event in sandbox_client._stub.ExecSandboxInteractive( + requests(), timeout=30 + ): + assert not exit_codes, "received another event after terminal exit" + payload = event.WhichOneof("payload") + if payload == "stdout": + stdout.extend(event.stdout.data) + elif payload == "stderr": + stderr.extend(event.stderr.data) + elif payload == "exit": + exit_codes.append(event.exit.exit_code) + assert stdout == b"drained-stdout" + assert stderr == b"drained-stderr" + assert exit_codes == [7] + + def test_list_scoped_and_for_all_workspaces( sandbox_client: SandboxClient, workspace_client: WorkspaceClient, diff --git a/sdk/go/README.md b/sdk/go/README.md index 6d695c6b4b..8e82aa2c34 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -278,6 +278,18 @@ consumers import a single package. See the [Architecture](https://ro14nd.de/open ## Features +Use `CloseInteractiveInput(session)` to close stdin and resize input while keeping +output readable. SDK sessions implement the optional `InteractiveSessionControl` +interface (`CloseWrite()` and `Cancel()`); the original `InteractiveSession` +interface remains unchanged for existing mocks and wrappers. Input closure returns +`ErrorUnimplemented` for sessions without that capability and leaves them open. +`CancelInteractive(session)` uses `Cancel()` when available and otherwise calls +`Close()`. SDK close/cancel operations are idempotent; writes and resizes after +input closure return `io.ErrClosedPipe`. Drain `Read` concurrently with waiting for `ExitCode()`. +`ExitCode()` waits for final gRPC status and returns any observed process exit code +alongside a later stream error. An exit event alone does not establish successful +stream completion. + | Feature | Interface | Docs | |---------|-----------|------| | Sandbox lifecycle (create, get, list, delete, watch, wait) | `SandboxInterface` | [Sandboxes](https://ro14nd.de/openshell-sdk-go/api/sandboxes.html) | diff --git a/sdk/go/docs/src/api/exec.md b/sdk/go/docs/src/api/exec.md index 0d088a6999..72ff49b347 100644 --- a/sdk/go/docs/src/api/exec.md +++ b/sdk/go/docs/src/api/exec.md @@ -139,8 +139,22 @@ type InteractiveSession interface { | `Read` | Reads output from the process into the provided buffer. | | `Write` | Sends input to the process. | | `Resize` | Updates the terminal dimensions (columns and rows). | -| `ExitCode` | Returns the process exit code after the session ends. | -| `Close` | Closes the session and releases resources. | +| `ExitCode` | Waits for final gRPC status; returns any observed exit code alongside a later stream error. | +| `Close` | Aborts the RPC and releases resources. | + +SDK sessions also implement the optional `InteractiveSessionControl` interface, +which embeds `InteractiveSession` and adds `CloseWrite() error` and `Cancel() error`. +Existing custom implementations only need the original methods above. + +Call `CloseInteractiveInput(session)` to end stdin and resize input without +cancelling output. It invokes `CloseWrite()` when supported; otherwise it returns +`ErrorUnimplemented` without closing the session. Call `CancelInteractive(session)` +to invoke `Cancel()` when supported, falling back to `Close()` for legacy sessions. +The SDK's close and cancel methods are idempotent. + +Drain `Read` concurrently with waiting for `ExitCode`; bounded output buffers can +otherwise prevent completion. After input closure, writes and resizes return +`io.ErrClosedPipe`. Request EOF is not equivalent to a terminal Ctrl-D keystroke. ## ExecChunk diff --git a/sdk/go/openshell/v1/exec.go b/sdk/go/openshell/v1/exec.go index 217a1dc9b3..f30975d15c 100644 --- a/sdk/go/openshell/v1/exec.go +++ b/sdk/go/openshell/v1/exec.go @@ -27,10 +27,40 @@ type InteractiveSession interface { Read(p []byte) (int, error) Write(p []byte) (int, error) Resize(cols, rows uint32) error + // ExitCode waits for final stream completion. An observed exit code is + // returned alongside any later transport error. Drain Read concurrently. ExitCode() (int, error) Close() error } +// InteractiveSessionControl adds optional input closure and cancellation to +// InteractiveSession without requiring existing implementations to add methods. +// Sessions returned by this SDK implement both interfaces. +type InteractiveSessionControl interface { + InteractiveSession + // CloseWrite ends stdin and resize input without cancelling output. + CloseWrite() error + // Cancel aborts the RPC. Close retains the same full-close behavior. + Cancel() error +} + +// CloseInteractiveInput ends input while preserving output when supported. +// Unsupported sessions return ErrorUnimplemented and are left open. +func CloseInteractiveInput(session InteractiveSession) error { + if closer, ok := session.(interface{ CloseWrite() error }); ok { + return closer.CloseWrite() + } + return &StatusError{Code: ErrorUnimplemented, Message: "interactive session does not support closing input independently"} +} + +// CancelInteractive aborts a session, falling back to the original Close contract. +func CancelInteractive(session InteractiveSession) error { + if canceler, ok := session.(interface{ Cancel() error }); ok { + return canceler.Cancel() + } + return session.Close() +} + // ExecInterface defines command execution operations on sandboxes. // Methods accept a sandbox name and resolve it to an ID internally. type ExecInterface interface { diff --git a/sdk/go/openshell/v1/exec_client.go b/sdk/go/openshell/v1/exec_client.go index 8bc9c803b4..1adc4c29df 100644 --- a/sdk/go/openshell/v1/exec_client.go +++ b/sdk/go/openshell/v1/exec_client.go @@ -11,6 +11,7 @@ import ( "github.com/NVIDIA/OpenShell/sdk/go/openshell/v1/internal/converter" pb "github.com/NVIDIA/OpenShell/sdk/go/proto/openshellv1" "google.golang.org/grpc" + "google.golang.org/grpc/status" ) type execClient struct { @@ -174,19 +175,21 @@ func (s *execStream) Close() error { // interactiveSession wraps a bidirectional streaming RPC into the InteractiveSession interface. // A background goroutine owns the Recv loop and routes events to dataCh (for Read) -// and exitCh (for ExitCode), preventing concurrent Recv calls on the stream. +// and publishes the final exit/status through done, preventing concurrent Recv calls. type interactiveSession struct { - stream grpc.BidiStreamingClient[pb.ExecSandboxInput, pb.ExecSandboxEvent] - cancel context.CancelFunc - sendMu sync.Mutex - dataCh chan []byte - exitCh chan int - done chan struct{} - errOnce sync.Once - err error - buf []byte - - exitMu sync.Mutex + stream grpc.BidiStreamingClient[pb.ExecSandboxInput, pb.ExecSandboxEvent] + cancel context.CancelFunc + sendMu sync.Mutex + inputClosed bool + inputCloseErr error + closeOnce sync.Once + closeErr error + dataCh chan []byte + done chan struct{} + errOnce sync.Once + err error + buf []byte + exitCode int hasExitCode bool } @@ -196,7 +199,6 @@ func newInteractiveSession(ctx context.Context, cancel context.CancelFunc, strea stream: stream, cancel: cancel, dataCh: make(chan []byte, 64), - exitCh: make(chan int, 1), done: make(chan struct{}), } go s.readLoop(ctx) @@ -208,6 +210,7 @@ func (s *interactiveSession) setErr(err error) { } func (s *interactiveSession) readLoop(ctx context.Context) { + defer s.cancel() defer close(s.dataCh) defer close(s.done) for { @@ -215,10 +218,18 @@ func (s *interactiveSession) readLoop(ctx context.Context) { if err != nil { if err != io.EOF { s.setErr(converter.FromGRPCError(err)) + } else if ctx.Err() != nil { + s.setErr(converter.FromGRPCError(status.FromContextError(ctx.Err()).Err())) + } else if !s.hasExitCode { + s.setErr(&StatusError{Code: ErrorInternal, Message: "stream ended without exit event"}) } return } + if s.hasExitCode { + s.setErr(&StatusError{Code: ErrorInternal, Message: "received event after exit"}) + return + } chunk, code, convErr := converter.ExecChunkFromEvent(ev) if convErr != nil { s.setErr(convErr) @@ -226,15 +237,14 @@ func (s *interactiveSession) readLoop(ctx context.Context) { } // nil chunk with no error means exit event if chunk == nil { - select { - case s.exitCh <- code: - default: - } - return + s.exitCode = code + s.hasExitCode = true + continue } select { case s.dataCh <- chunk.Data: case <-ctx.Done(): + s.setErr(converter.FromGRPCError(status.FromContextError(ctx.Err()).Err())) return } } @@ -264,6 +274,9 @@ func (s *interactiveSession) Read(p []byte) (int, error) { func (s *interactiveSession) Write(p []byte) (int, error) { s.sendMu.Lock() defer s.sendMu.Unlock() + if s.inputClosed { + return 0, io.ErrClosedPipe + } err := s.stream.Send(&pb.ExecSandboxInput{ Payload: &pb.ExecSandboxInput_Stdin{Stdin: p}, }) @@ -276,6 +289,9 @@ func (s *interactiveSession) Write(p []byte) (int, error) { func (s *interactiveSession) Resize(cols, rows uint32) error { s.sendMu.Lock() defer s.sendMu.Unlock() + if s.inputClosed { + return io.ErrClosedPipe + } err := s.stream.Send(&pb.ExecSandboxInput{ Payload: &pb.ExecSandboxInput_Resize{ Resize: &pb.ExecSandboxWindowResize{ @@ -291,41 +307,33 @@ func (s *interactiveSession) Resize(cols, rows uint32) error { } func (s *interactiveSession) ExitCode() (int, error) { - s.exitMu.Lock() + <-s.done if s.hasExitCode { - code := s.exitCode - s.exitMu.Unlock() - return code, nil - } - s.exitMu.Unlock() - - select { - case code := <-s.exitCh: - s.exitMu.Lock() - s.exitCode = code - s.hasExitCode = true - s.exitMu.Unlock() - return code, nil - case <-s.done: - select { - case code := <-s.exitCh: - s.exitMu.Lock() - s.exitCode = code - s.hasExitCode = true - s.exitMu.Unlock() - return code, nil - default: - if s.err != nil { - return -1, s.err - } - return -1, &StatusError{Code: ErrorInternal, Message: "stream ended without exit event"} - } + return s.exitCode, s.err + } + return -1, s.err +} + +func (s *interactiveSession) CloseWrite() error { + s.sendMu.Lock() + defer s.sendMu.Unlock() + if !s.inputClosed { + s.inputClosed = true + s.inputCloseErr = s.stream.CloseSend() } + return s.inputCloseErr +} + +func (s *interactiveSession) Cancel() error { + s.closeOnce.Do(func() { + // Cancel before taking sendMu so a blocked Write can release it. + s.cancel() + s.closeErr = s.CloseWrite() + <-s.done + }) + return s.closeErr } func (s *interactiveSession) Close() error { - s.cancel() - err := s.stream.CloseSend() - <-s.done - return err + return s.Cancel() } diff --git a/sdk/go/openshell/v1/exec_client_test.go b/sdk/go/openshell/v1/exec_client_test.go index 5489a1f95f..8354fb115c 100644 --- a/sdk/go/openshell/v1/exec_client_test.go +++ b/sdk/go/openshell/v1/exec_client_test.go @@ -85,6 +85,8 @@ type mockExecServer struct { interactiveErr error interactiveWaitInput bool interactiveBlock bool + interactiveWaitEOF bool + interactiveFinalErr error receivedInputs []*pb.ExecSandboxInput } @@ -116,6 +118,8 @@ func (s *mockExecServer) ExecSandboxInteractive(stream grpc.BidiStreamingServer[ s.mu.Lock() interactiveErr := s.interactiveErr interactiveBlock := s.interactiveBlock + waitEOF := s.interactiveWaitEOF + finalErr := s.interactiveFinalErr events := make([]*pb.ExecSandboxEvent, len(s.interactiveEvents)) copy(events, s.interactiveEvents) s.mu.Unlock() @@ -136,6 +140,17 @@ func (s *mockExecServer) ExecSandboxInteractive(stream grpc.BidiStreamingServer[ <-stream.Context().Done() return stream.Context().Err() } + if waitEOF { + for { + _, recvErr := stream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + return recvErr + } + } + } s.mu.Lock() waitInput := s.interactiveWaitInput @@ -170,7 +185,7 @@ func (s *mockExecServer) ExecSandboxInteractive(stream grpc.BidiStreamingServer[ return err } } - return nil + return finalErr } func setupExecTest(t *testing.T, mock *mockExecServer) (*execClient, func()) { @@ -407,6 +422,86 @@ func TestExecInteractive_CloseCancelsReceiveStream(t *testing.T) { } } +func TestExecInteractive_CloseWriteDrainsOutput(t *testing.T) { + mock := newMockExecServer() + mock.interactiveWaitEOF = true + mock.interactiveEvents = []*pb.ExecSandboxEvent{ + {Payload: &pb.ExecSandboxEvent_Stdout{Stdout: &pb.ExecSandboxStdout{Data: []byte("after EOF")}}}, + {Payload: &pb.ExecSandboxEvent_Exit{Exit: &pb.ExecSandboxExit{ExitCode: 7}}}, + } + client, cleanup := setupExecTest(t, mock) + defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + session, err := client.Interactive(ctx, "default", "test-sandbox", []string{"cat"}, 0, 0) + require.NoError(t, err) + defer func() { _ = session.Close() }() + require.Implements(t, (*InteractiveSessionControl)(nil), session) + require.NoError(t, CloseInteractiveInput(session)) + require.NoError(t, CloseInteractiveInput(session)) + _, err = session.Write([]byte("late")) + require.ErrorIs(t, err, io.ErrClosedPipe) + require.ErrorIs(t, session.Resize(80, 24), io.ErrClosedPipe) + output, err := io.ReadAll(session) + require.NoError(t, err) + require.Equal(t, "after EOF", string(output)) + code, err := session.ExitCode() + require.NoError(t, err) + require.Equal(t, 7, code) +} + +func TestExecInteractive_RetainsExitOnTerminalFailure(t *testing.T) { + for _, finalErr := range []error{status.Error(codes.Unavailable, "connection lost"), status.Error(codes.Canceled, "cancelled")} { + t.Run(status.Code(finalErr).String(), func(t *testing.T) { + mock := newMockExecServer() + mock.interactiveEvents = []*pb.ExecSandboxEvent{ + {Payload: &pb.ExecSandboxEvent_Exit{Exit: &pb.ExecSandboxExit{ExitCode: 7}}}, + } + mock.interactiveFinalErr = finalErr + client, cleanup := setupExecTest(t, mock) + defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + session, err := client.Interactive(ctx, "default", "test-sandbox", []string{"true"}, 0, 0) + require.NoError(t, err) + defer func() { _ = session.Close() }() + code, err := session.ExitCode() + require.Error(t, err) + require.Equal(t, 7, code) + var sdkErr *StatusError + require.ErrorAs(t, err, &sdkErr) + require.Equal(t, status.Code(finalErr), status.Code(sdkErr.Cause)) + codeAgain, errAgain := session.ExitCode() + require.Equal(t, code, codeAgain) + require.Equal(t, err, errAgain) + }) + } +} + +func TestExecInteractive_RejectsEventsAfterExit(t *testing.T) { + for name, event := range map[string]*pb.ExecSandboxEvent{ + "duplicate exit": {Payload: &pb.ExecSandboxEvent_Exit{Exit: &pb.ExecSandboxExit{ExitCode: 1}}}, + "late output": {Payload: &pb.ExecSandboxEvent_Stdout{Stdout: &pb.ExecSandboxStdout{Data: []byte("late")}}}, + } { + t.Run(name, func(t *testing.T) { + mock := newMockExecServer() + mock.interactiveEvents = []*pb.ExecSandboxEvent{ + {Payload: &pb.ExecSandboxEvent_Exit{Exit: &pb.ExecSandboxExit{ExitCode: 0}}}, event, + } + client, cleanup := setupExecTest(t, mock) + defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + session, err := client.Interactive(ctx, "default", "test-sandbox", []string{"true"}, 0, 0) + require.NoError(t, err) + defer func() { _ = session.Close() }() + code, err := session.ExitCode() + require.Equal(t, 0, code) + require.ErrorContains(t, err, "after exit") + }) + } +} + func TestExecInteractive_Write(t *testing.T) { mock := newMockExecServer() mock.interactiveWaitInput = true diff --git a/sdk/go/openshell/v1/exec_control_test.go b/sdk/go/openshell/v1/exec_control_test.go new file mode 100644 index 0000000000..3e2392a821 --- /dev/null +++ b/sdk/go/openshell/v1/exec_control_test.go @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1 + +import ( + "errors" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +// Deliberately implements only the original interface: downstream mocks must +// continue compiling without adding the optional lifecycle methods. +type legacyInteractiveSession struct { + closed bool + err error +} + +var _ InteractiveSession = (*legacyInteractiveSession)(nil) + +func (*legacyInteractiveSession) Read([]byte) (int, error) { return 0, io.EOF } +func (*legacyInteractiveSession) Write(p []byte) (int, error) { return len(p), nil } +func (*legacyInteractiveSession) Resize(uint32, uint32) error { return nil } +func (*legacyInteractiveSession) ExitCode() (int, error) { return 0, nil } +func (s *legacyInteractiveSession) Close() error { s.closed = true; return s.err } + +type controlledInteractiveSession struct { + legacyInteractiveSession + inputClosed bool + cancelled bool +} + +func (s *controlledInteractiveSession) CloseWrite() error { s.inputClosed = true; return s.err } +func (s *controlledInteractiveSession) Cancel() error { s.cancelled = true; return s.err } + +func TestInteractiveControlLegacyCompatibility(t *testing.T) { + s := &legacyInteractiveSession{err: errors.New("close error")} + var statusErr *StatusError + require.ErrorAs(t, CloseInteractiveInput(s), &statusErr) + require.Equal(t, ErrorUnimplemented, statusErr.Code) + require.False(t, s.closed, "unsupported input closure must not cancel the session") + require.ErrorIs(t, CancelInteractive(s), s.err) + require.True(t, s.closed) +} + +func TestInteractiveControlDispatch(t *testing.T) { + s := &controlledInteractiveSession{legacyInteractiveSession: legacyInteractiveSession{err: errors.New("control error")}} + require.Implements(t, (*InteractiveSessionControl)(nil), s) + require.ErrorIs(t, CloseInteractiveInput(s), s.err) + require.True(t, s.inputClosed) + require.ErrorIs(t, CancelInteractive(s), s.err) + require.True(t, s.cancelled) + require.False(t, s.closed, "explicit cancellation takes precedence over fallback") +} diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 8237e91c57..99c4fac9d4 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -126,7 +126,20 @@ for await (const event of client.sandbox.execStream(name, ['pytest', '-q'])) { } ``` -`execInteractive` is the TTY + stdin transport primitive. Drive it by consuming `output`, which yields the same chunk/exit events; `done` resolves with the exit code once the stream reaches its exit event and rejects if it ends without one. It ships raw bytes only; raw mode, signal forwarding, and SIGWINCH stay with the caller. +`execInteractive` is the TTY + stdin transport primitive. Consume `output` concurrently with awaiting `done`. The helper yields its terminal exit event and resolves `done` only after receiving an exit event and successful final gRPC status. If transport completion fails, `done` rejects and `session.exitCode` retains any observed process exit code. Duplicate exit events and output after exit are errors. + +The RPC starts immediately when the session is created, even before you consume +`output`. A background receiver observes transport errors and buffers up to 1 MiB +of output in 64 KiB chunks. Once the queue fills, receiving waits for the caller +to drain it; cancellation interrupts that wait. The bound excludes the transport's +current response frame and its own buffers. + +Call `closeInput()` (or its compatibility alias `close()`) to end stdin and resize input while continuing to receive output. Later writes and resizes throw. Call `cancel()` to abort the RPC. Both close and cancel are idempotent. The helper ships raw bytes only; raw mode, signal forwarding, and SIGWINCH stay with the caller. Input closure is not a terminal Ctrl-D keystroke. + +`execInteractive()` returns `ExecInteractiveSessionControl`, which extends the +original `ExecInteractiveSession` with `closeInput()`, `cancel()`, and `exitCode`. +Existing mocks and wrappers can keep implementing the original interface without +adding those members. Use the extended type when a wrapper exposes the new controls. ```ts const session = await client.sandbox.execInteractive(name, ['bash']) @@ -284,6 +297,7 @@ mise run sdk:ts:lint # Biome: lint + format check (read-only) mise run sdk:ts:typecheck # tsc --noEmit mise run sdk:ts:test # Vitest unit tests with an 80% line-coverage gate mise run sdk:ts:build # emit dist/ +mise run e2e:sdk:ts:exec # public interactive helper against an isolated Docker gateway ``` Formatting and linting are handled by [Biome](https://biomejs.dev) (`biome.json`): 2-space indent, single quotes, semicolons, 120-column width. Generated `src/gen/` is excluded. `sdk:ts:lint` runs in CI as part of `sdk:ts:ci`. diff --git a/sdk/typescript/e2e/interactive-exec.mjs b/sdk/typescript/e2e/interactive-exec.mjs new file mode 100644 index 0000000000..eaeb94f5f1 --- /dev/null +++ b/sdk/typescript/e2e/interactive-exec.mjs @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Run through mise run e2e:sdk:ts:exec, which supplies an isolated gateway. +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { SandboxClient } from '../dist/index.js'; + +test('public interactive helper drains output after input EOF and verifies completion', { + timeout: 300_000, +}, async () => { + assert.ok(process.env.XDG_CONFIG_HOME, 'run with the Docker gateway wrapper'); + assert.ok(process.env.OPENSHELL_GATEWAY, 'run with the Docker gateway wrapper'); + const dir = join(process.env.XDG_CONFIG_HOME, 'openshell', 'gateways', process.env.OPENSHELL_GATEWAY); + const metadata = JSON.parse(readFileSync(join(dir, 'metadata.json'), 'utf8')); + const client = await SandboxClient.connect({ + gateway: metadata.gateway_endpoint, + ...(metadata.gateway_endpoint.startsWith('https:') + ? { + caCert: readFileSync(join(dir, 'mtls', 'ca.crt')), + clientCert: readFileSync(join(dir, 'mtls', 'tls.crt')), + clientKey: readFileSync(join(dir, 'mtls', 'tls.key')), + } + : {}), + }); + const name = `ts-eof-${Date.now().toString(36)}`; + await client.create({ + name, + image: process.env.OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE ?? 'ghcr.io/nvidia/openshell-community/sandboxes/base:latest', + }); + try { + await client.waitReady(name, 180); + const session = await client.execInteractive( + name, + ['/bin/sh', '-c', 'input=$(cat); printf "stdout:%s" "$input"; printf "stderr:drained" >&2; exit 7'], + { tty: false, timeoutSecs: 30, signal: AbortSignal.timeout(45_000) }, + ); + try { + // cat cannot finish until closeInput reaches the sandbox. All command + // output therefore proves that request EOF left the response open. + session.write(Buffer.from('input-before-eof')); + session.closeInput(); + session.closeInput(); // The public control remains idempotent. + const stdout = []; + const stderr = []; + const exits = []; + for await (const event of session.output) { + assert.equal(exits.length, 0, 'exit must be the final application event'); + if ('type' in event) exits.push(event.exitCode); + else (event.stream === 'stdout' ? stdout : stderr).push(event.data); + } + assert.equal(Buffer.concat(stdout).toString(), 'stdout:input-before-eof'); + assert.equal(Buffer.concat(stderr).toString(), 'stderr:drained'); + assert.deepEqual(exits, [7]); + // A nonzero process exit is not a failed transport. done resolves only + // after the helper has verified the final gRPC status. + assert.equal(await session.done, 7); + assert.equal(session.exitCode, 7); + } finally { + session.cancel(); + } + } finally { + await client.delete(name); + } +}); diff --git a/sdk/typescript/src/client.test.ts b/sdk/typescript/src/client.test.ts index 60d93dbe2d..dcc85f328e 100644 --- a/sdk/typescript/src/client.test.ts +++ b/sdk/typescript/src/client.test.ts @@ -24,6 +24,7 @@ import { } from './client.js'; import { OpenShell, SandboxPhase, ServiceStatus } from './gen/openshell_pb.js'; import { PolicySource, SettingScope } from './gen/sandbox_pb.js'; +import type { ExecInteractiveSession, ExecInteractiveSessionControl } from './index.js'; function client(impl: Partial>): SandboxClient { const transport: Transport = createRouterTransport((router) => { @@ -956,6 +957,35 @@ describe('Pushable', () => { }); describe('execInteractive', () => { + it('preserves legacy session implementations and exposes SDK lifecycle controls', async () => { + // These are exactly the original required members, checked through the + // public package exports so downstream wrappers can keep their old types. + const legacy: ExecInteractiveSession = { + output: (async function* () { + yield { type: 'exit' as const, exitCode: 0 }; + })(), + write() {}, + resize() {}, + close() {}, + done: Promise.resolve(0), + }; + const wrap = (session: ExecInteractiveSession): ExecInteractiveSession => session; + expect(wrap(legacy)).toBe(legacy); + + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: async function* () { + yield { payload: { case: 'exit', value: { exitCode: 0 } } }; + }, + }); + const controlled: ExecInteractiveSessionControl = await sandbox.execInteractive('sb', ['true']); + expect(wrap(controlled)).toBe(controlled); + controlled.closeInput(); + expect(await controlled.done).toBe(0); + expect(controlled.exitCode).toBe(0); + controlled.cancel(); + }); + it('sends start first with tty/cols/rows, streams output, and resolves done', async () => { const cases: string[] = []; let started: (ScopedRequest & { tty?: boolean; cols?: number; rows?: number }) | undefined; @@ -1009,6 +1039,184 @@ describe('execInteractive', () => { }); describe('exec done settlement', () => { + it('starts the command and observes completion before output is consumed', async () => { + let started = false; + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: async function* () { + started = true; + yield { payload: { case: 'stdout', value: { data: enc('started') } } }; + yield { payload: { case: 'exit', value: { exitCode: 0 } } }; + }, + }); + const session = await sandbox.execInteractive('sb', ['bash']); + await expect.poll(() => started).toBe(true); + expect(await session.done).toBe(0); + const events = []; + for await (const event of session.output) events.push(event); + expect(events).toEqual([ + { stream: 'stdout', data: Buffer.from('started') }, + { type: 'exit', exitCode: 0 }, + ]); + }); + + it('observes early transport failures without output consumption', async () => { + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: () => { + throw new ConnectError('early failure', Code.Unavailable); + }, + }); + const session = await sandbox.execInteractive('sb', ['bash']); + await expect(session.done).rejects.toMatchObject({ connectCode: Code.Unavailable }); + const iterator = session.output[Symbol.asyncIterator](); + await expect(iterator.next()).rejects.toMatchObject({ connectCode: Code.Unavailable }); + }); + + it('cancels a receiver blocked by output backpressure', async () => { + let released = false; + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: async function* (_requests, ctx) { + ctx.signal.addEventListener('abort', () => { + released = true; + }); + // More than the SDK queue budget, in one transport event. + yield { payload: { case: 'stdout', value: { data: new Uint8Array(4 * 1024 * 1024) } } }; + yield { payload: { case: 'exit', value: { exitCode: 0 } } }; + }, + }); + const session = await sandbox.execInteractive('sb', ['bash']); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(session.exitCode).toBeUndefined(); + session.cancel(); + await expect(session.done).rejects.toMatchObject({ code: 'canceled' }); + await expect.poll(() => released).toBe(true); + await expect(session.output[Symbol.asyncIterator]().next()).rejects.toMatchObject({ code: 'canceled' }); + }); + + it('drains output larger than the queue budget without losing bytes', async () => { + const data = Buffer.alloc(2 * 1024 * 1024 + 7, 'x'); + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: async function* () { + yield { payload: { case: 'stdout', value: { data } } }; + yield { payload: { case: 'stderr', value: { data: enc('last') } } }; + yield { payload: { case: 'exit', value: { exitCode: 3 } } }; + }, + }); + const session = await sandbox.execInteractive('sb', ['bash']); + const chunks: Buffer[] = []; + for await (const event of session.output) { + if ('type' in event) expect(await session.done).toBe(3); + else chunks.push(event.data); + } + const actual = Buffer.concat(chunks); + const expected = Buffer.concat([data, Buffer.from('last')]); + expect(actual.length).toBe(expected.length); + // Compare bytes natively: deep equality on a multi-MiB Buffer can exhaust + // the test timeout on CI even when the stream drains promptly. + expect(actual.equals(expected)).toBe(true); + }); + + it('retains the exit code but rejects completion on a later transport error', async () => { + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: async function* () { + yield { payload: { case: 'exit', value: { exitCode: 7 } } }; + throw new ConnectError('connection lost after exit', Code.Unavailable); + }, + }); + const session = await sandbox.execInteractive('sb', ['bash']); + await expect( + (async () => { + for await (const _event of session.output) { + /* drain through trailers */ + } + })(), + ).rejects.toMatchObject({ connectCode: Code.Unavailable }); + await expect(session.done).rejects.toMatchObject({ connectCode: Code.Unavailable }); + expect(session.exitCode).toBe(7); + }); + + it.each(['stdout', 'exit'] as const)('rejects %s after an exit event', async (payload) => { + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: async function* () { + yield { payload: { case: 'exit', value: { exitCode: 0 } } }; + if (payload === 'stdout') yield { payload: { case: 'stdout', value: { data: enc('late') } } }; + else yield { payload: { case: 'exit', value: { exitCode: 1 } } }; + }, + }); + const session = await sandbox.execInteractive('sb', ['bash']); + await expect( + (async () => { + for await (const _event of session.output) { + /* drain */ + } + })(), + ).rejects.toThrow('after exit'); + await expect(session.done).rejects.toThrow('after exit'); + expect(session.exitCode).toBe(0); + }); + + it('closes input idempotently and rejects later stdin and resize', async () => { + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: async function* (requests) { + for await (const _input of requests) { + /* wait for request EOF */ + } + yield { payload: { case: 'stdout', value: { data: enc('drained') } } }; + yield { payload: { case: 'exit', value: { exitCode: 0 } } }; + }, + }); + const session = await sandbox.execInteractive('sb', ['bash']); + session.closeInput(); + session.close(); + expect(() => session.write(Buffer.from('late'))).toThrow('input is closed'); + expect(() => session.resize(80, 24)).toThrow('input is closed'); + const output = []; + for await (const event of session.output) output.push(event); + expect(output).toHaveLength(2); + expect(await session.done).toBe(0); + }); + + it('cancel settles completion even before output is consumed', async () => { + const sandbox = client({ getSandbox: () => readySandbox('sb', 'sb-id') }); + const session = await sandbox.execInteractive('sb', ['bash']); + session.cancel(); + session.cancel(); + await expect(session.done).rejects.toMatchObject({ code: 'canceled' }); + expect(() => session.write(Buffer.from('late'))).toThrow('input is closed'); + }); + + it('external cancellation settles completion before output is consumed', async () => { + const controller = new AbortController(); + const sandbox = client({ getSandbox: () => readySandbox('sb', 'sb-id') }); + const session = await sandbox.execInteractive('sb', ['bash'], { signal: controller.signal }); + controller.abort(); + await expect(session.done).rejects.toMatchObject({ code: 'canceled' }); + expect(() => session.resize(80, 24)).toThrow('input is closed'); + }); + + it('external cancellation settles completion while output is paused', async () => { + const controller = new AbortController(); + const sandbox = client({ + getSandbox: () => readySandbox('sb', 'sb-id'), + execSandboxInteractive: async function* () { + yield { payload: { case: 'stdout', value: { data: enc('partial') } } }; + yield { payload: { case: 'exit', value: { exitCode: 0 } } }; + }, + }); + const session = await sandbox.execInteractive('sb', ['bash'], { signal: controller.signal }); + const iterator = session.output[Symbol.asyncIterator](); + await iterator.next(); + controller.abort(); + await expect(session.done).rejects.toMatchObject({ code: 'canceled' }); + await iterator.return?.(); + }); + it('resolves done even when the consumer breaks right after the exit event', async () => { const sandbox = client({ getSandbox: () => readySandbox('sb', 'sb-id'), @@ -1022,7 +1230,7 @@ describe('exec done settlement', () => { for await (const event of session.output) { if ('type' in event) break; // break on exit: the generator never resumes } - // Without settling `done` before the exit yield, this would hang forever. + // The public exit is yielded only after successful terminal status. expect(await session.done).toBe(3); }); diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts index 549a5e6021..b0595f0c0a 100644 --- a/sdk/typescript/src/client.ts +++ b/sdk/typescript/src/client.ts @@ -299,16 +299,28 @@ export interface ExecInteractiveOptions extends SandboxWorkspaceOptions { // The transport half of an interactive exec: raw stdin/stdout/stderr plus // resize, with no terminal glue. Drive it by consuming `output`, which yields -// chunks then a terminal exit event; `done` resolves with the exit code once -// the stream reaches that exit event and rejects if the stream ends without one. +// chunks then a terminal exit event; `done` resolves only after an exit event +// and successful RPC completion. Consume output concurrently with awaiting done. export interface ExecInteractiveSession { output: AsyncIterable; write(data: Buffer): void; resize(cols: number, rows: number): void; + /** Close stdin and resize input while preserving output. */ close(): void; done: Promise; } +/** Lifecycle controls available on SDK-created sessions. The base interface + * retains its original members for existing custom sessions and wrappers. */ +export interface ExecInteractiveSessionControl extends ExecInteractiveSession { + /** Close stdin and resize input, preserving output until completion. */ + closeInput(): void; + /** Cancel the RPC and stop receiving output. */ + cancel(): void; + /** Observed process exit, retained even if final RPC completion fails. */ + readonly exitCode: number | undefined; +} + /** Cancellation for the poll-based wait helpers. */ export interface WaitOptions extends SandboxWorkspaceOptions { /** Abort the wait (and the in-flight poll RPC) early. */ @@ -709,6 +721,60 @@ export class Pushable implements AsyncIterable { } } +// Single producer/consumer queue. Closing wakes both directions, including a +// producer blocked by backpressure. Successful completion drains queued values. +class ExecOutputQueue { + private readonly values: ExecStreamEvent[] = []; + private ended = false; + private error: unknown; + private reader?: () => void; + private writer?: () => void; + + async push(value: ExecStreamEvent): Promise { + // The terminal exit carries no output bytes and must never block cleanup + // after done has already settled and its cancellation listener is removed. + while (!('type' in value) && this.values.length >= 16 && !this.ended) { + await new Promise((resolve) => { + this.writer = resolve; + }); + } + if (this.ended) throw this.error ?? new SdkError('canceled', 'exec output closed'); + this.values.push(value); + this.reader?.(); + this.reader = undefined; + } + + end(error?: unknown, discard = false): void { + if (discard) this.values.length = 0; + if (!this.ended) { + this.ended = true; + this.error = error; + } + this.reader?.(); + this.writer?.(); + this.reader = undefined; + this.writer = undefined; + } + + async *[Symbol.asyncIterator](): AsyncGenerator { + for (;;) { + const value = this.values.shift(); + if (value !== undefined) { + this.writer?.(); + this.writer = undefined; + yield value; + } else if (this.ended) { + if (this.error !== undefined) throw this.error; + return; + } else { + await new Promise((resolve) => { + this.reader = resolve; + }); + } + } + } +} + /** One response page from a list operation. */ export interface Page { readonly items: T[]; @@ -1112,7 +1178,7 @@ export class SandboxClient { name: string, command: string[], options?: ExecInteractiveOptions | null, - ): Promise { + ): Promise { try { await this.get(name, { workspace: options?.workspace, ...(options?.signal ? { signal: options.signal } : {}) }); } catch (e) { @@ -1138,7 +1204,19 @@ export class SandboxClient { }, }); - const stream = this.grpc.execSandboxInteractive(input, { signal: options?.signal }); + const controller = new AbortController(); + const signal = options?.signal ? AbortSignal.any([options.signal, controller.signal]) : controller.signal; + const grpc = this.grpc; + const queue = new ExecOutputQueue(); + let inputClosed = false; + let exitCode: number | undefined; + const closeInput = (): void => { + inputClosed = true; + input.end(); + }; + const assertInputOpen = (): void => { + if (inputClosed || signal.aborted) throw new SdkError('io', 'exec input is closed'); + }; let resolveDone!: (code: number) => void; let rejectDone!: (err: unknown) => void; const done = new Promise((resolve, reject) => { @@ -1149,72 +1227,113 @@ export class SandboxClient { // keeps an unobserved rejection from surfacing as an unhandledRejection; // real awaiters still receive it through their own handler. void done.catch(() => {}); - // Settle exactly once. The exit code wins; error/abandonment only apply - // when no exit was observed. + // The process exit and the terminal transport status are separate outcomes. let settled = false; const settleExit = (code: number): void => { if (settled) return; settled = true; + signal.removeEventListener('abort', onAbort); resolveDone(code); }; const settleError = (err: unknown): void => { if (settled) return; settled = true; + signal.removeEventListener('abort', onAbort); rejectDone(err); }; - async function* output(): AsyncGenerator { - let sawExit = false; + const onAbort = (): void => { + closeInput(); + const error = new SdkError('canceled', 'exec cancelled'); + queue.end(error, true); + settleError(error); + }; + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + + async function receive(): Promise { try { + if (signal.aborted) throw new SdkError('canceled', 'exec cancelled'); + // Start and observe the transport immediately, independently of output + // consumption. Backpressure bounds the queue to 16 chunks of 64 KiB. + const stream = grpc.execSandboxInteractive(input, { signal }); for await (const event of stream) { + if (signal.aborted) throw new SdkError('canceled', 'exec cancelled'); + if (exitCode !== undefined) { + throw new SdkError('rpc', 'ExecSandboxInteractive received an event after exit'); + } switch (event.payload.case) { case 'stdout': - yield { - stream: 'stdout', - data: Buffer.from(event.payload.value.data), - }; - break; case 'stderr': - yield { - stream: 'stderr', - data: Buffer.from(event.payload.value.data), - }; + for (let offset = 0; offset < event.payload.value.data.length; offset += 64 * 1024) { + await queue.push({ + stream: event.payload.case, + data: Buffer.from(event.payload.value.data.subarray(offset, offset + 64 * 1024)), + }); + } break; case 'exit': - sawExit = true; - // Settle `done` before yielding: a consumer that breaks on the - // exit event abandons the generator at the yield, so anything - // after it would never run. - settleExit(event.payload.value.exitCode); - yield { type: 'exit', exitCode: event.payload.value.exitCode }; + exitCode = event.payload.value.exitCode; + closeInput(); break; + default: + throw new SdkError('rpc', 'ExecSandboxInteractive received an empty or unknown event'); } } - if (!sawExit) { + if (exitCode === undefined) { throw new SdkError('rpc', 'ExecSandboxInteractive stream ended without an exit event'); } + if (signal.aborted) throw new SdkError('canceled', 'exec cancelled before completion'); + // Delay the public exit event until trailers have been consumed. A + // caller can still break on exit without losing the terminal status. + settleExit(exitCode); + await queue.push({ type: 'exit', exitCode }); + queue.end(); } catch (e) { const err = e instanceof SdkError ? e : fromConnect(e); settleError(err); - throw err; + queue.end(err); + } finally { + closeInput(); + controller.abort(); + } + } + + // receive catches transport failures even when nobody consumes output/done. + const receiving = receive(); + async function* output(): AsyncGenerator { + try { + yield* queue; } finally { - input.end(); - // Consumer abandoned the stream before an exit event (early break or - // return): settle `done` so it can never hang. - settleError(new SdkError('rpc', 'exec output abandoned before exit')); + const error = new SdkError('rpc', 'exec output abandoned before completion'); + settleError(error); + queue.end(error, true); + closeInput(); + controller.abort(); + await receiving; } } return { output: output(), write(data: Buffer): void { + assertInputOpen(); input.push({ payload: { case: 'stdin', value: new Uint8Array(data) } }); }, resize(cols: number, rows: number): void { + assertInputOpen(); input.push({ payload: { case: 'resize', value: { cols, rows } } }); }, - close(): void { - input.end(); + closeInput, + close: closeInput, + cancel(): void { + closeInput(); + queue.end(new SdkError('canceled', 'exec cancelled'), true); + controller.abort(); + settleError(new SdkError('canceled', 'exec cancelled')); + }, + get exitCode(): number | undefined { + return exitCode; }, done, }; diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 3902f9135a..e0802fc8f7 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -12,6 +12,7 @@ export type { ExecExitEvent, ExecInteractiveOptions, ExecInteractiveSession, + ExecInteractiveSessionControl, ExecOptions, ExecResult, ExecStreamChunk, diff --git a/tasks/typescript.toml b/tasks/typescript.toml index 57277e05ba..ed07379c63 100644 --- a/tasks/typescript.toml +++ b/tasks/typescript.toml @@ -59,6 +59,11 @@ depends = ["sdk:ts:proto"] dir = "sdk/typescript" run = "npm test" +["e2e:sdk:ts:exec"] +description = "Test public TypeScript interactive exec against a Docker-backed gateway" +depends = ["sdk:ts:build"] +run = "e2e/with-docker-gateway.sh node --test sdk/typescript/e2e/interactive-exec.mjs" + ["sdk:ts:ci"] description = "TypeScript SDK checks (proto lint, Biome lint, codegen, typecheck, test, build)" depends = [ From f8b1fd8b57d353796e0a8cfae6131b201427d0db Mon Sep 17 00:00:00 2001 From: krishicks Date: Tue, 22 Sep 2026 16:42:15 +0000 Subject: [PATCH 03/10] fix(supervisor): restore OCSF schema downgrade (#3554) The RFC-0012 architecture migration preserved sandbox OCSF JSON enablement but introduced a regression: ocsf_schema_version was no longer honored. This fixes the regression so that schema downgrades occur again. Signed-off-by: Kris Hicks --- crates/openshell-supervisor/src/lib.rs | 71 +++++++++++++++++++++++++ crates/openshell-supervisor/src/main.rs | 6 ++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 4a48c0704a..a6e029725b 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -569,6 +569,7 @@ pub async fn run_sandbox( ssh_socket_path: Option, health_socket_path: Option, ocsf_enabled: Arc, + ocsf_schema_version: Arc>, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, auth_bundle: openshell_core::jwt::SupervisorAuthBundle, @@ -1011,6 +1012,7 @@ pub async fn run_sandbox( let poll_endpoint = endpoint.to_string(); let poll_engine = engine.clone(); let poll_ocsf_enabled = ocsf_enabled.clone(); + let poll_ocsf_schema_version = ocsf_schema_version.clone(); let poll_pid = entrypoint_pid.clone(); let poll_provider_credentials = provider_credentials.clone(); let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); @@ -1031,6 +1033,7 @@ pub async fn run_sandbox( entrypoint_pid: poll_pid, interval_secs: poll_interval_secs, ocsf_enabled: poll_ocsf_enabled, + ocsf_schema_version: poll_ocsf_schema_version, provider_credentials: poll_provider_credentials, provider_readiness: provider_readiness.clone(), policy_local_ctx: poll_policy_local, @@ -3513,6 +3516,7 @@ struct PolicyPollLoopContext { entrypoint_pid: Arc, interval_secs: u64, ocsf_enabled: Arc, + ocsf_schema_version: Arc>, provider_credentials: ProviderCredentialState, provider_readiness: ProviderReadinessTracker, policy_local_ctx: Option>, @@ -4034,6 +4038,7 @@ async fn run_policy_poll_loop_with_client( ); current_policy_generation = Some(generation.clone()); apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); apply_agent_proposals_enabled( &ctx.agent_proposals, agent_proposals_enabled_from_settings(&result.settings), @@ -4072,6 +4077,7 @@ async fn run_policy_poll_loop_with_client( } (InitialPollDisposition::TrackOnly, _) => { apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); apply_agent_proposals_enabled( &ctx.agent_proposals, agent_proposals_enabled_from_settings(&result.settings), @@ -4680,6 +4686,7 @@ async fn run_policy_poll_loop_with_client( // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); // Apply the agent-proposals feature toggle. On a false→true transition // we lazily install the skill so a sandbox that started with the flag @@ -4738,6 +4745,38 @@ fn extract_bool_setting( }) } +fn apply_ocsf_schema_version_setting( + version: &std::sync::Mutex, + settings: &std::collections::HashMap, +) { + let new_version = extract_string_setting(settings, "ocsf_schema_version").unwrap_or_default(); + if let Ok(mut current) = version.lock() + && *current != new_version + { + info!( + ocsf_schema_version = %new_version, + "OCSF schema version target changed" + ); + *current = new_version; + } +} + +/// Extract a string value from an effective setting, if present. +fn extract_string_setting( + settings: &std::collections::HashMap, + key: &str, +) -> Option { + use openshell_core::proto::setting_value; + settings + .get(key) + .and_then(|es| es.value.as_ref()) + .and_then(|sv| sv.value.as_ref()) + .and_then(|v| match v { + setting_value::Value::StringValue(value) => Some(value.clone()), + _ => None, + }) +} + fn agent_proposals_enabled_from_settings( settings: &std::collections::HashMap, ) -> bool { @@ -5061,6 +5100,37 @@ mod tests { assert!(!enabled.load(Ordering::Relaxed)); } + #[test] + fn apply_ocsf_schema_version_setting_updates_from_initial_settings_snapshot() { + let version = std::sync::Mutex::new(String::new()); + let mut settings = std::collections::HashMap::new(); + settings.insert( + "ocsf_schema_version".to_string(), + openshell_core::proto::EffectiveSetting { + value: Some(openshell_core::proto::SettingValue { + value: Some(openshell_core::proto::setting_value::Value::StringValue( + "1.3".into(), + )), + }), + scope: openshell_core::proto::SettingScope::Sandbox.into(), + }, + ); + + apply_ocsf_schema_version_setting(&version, &settings); + + assert_eq!(*version.lock().unwrap(), "1.3"); + } + + #[test] + fn apply_ocsf_schema_version_setting_clears_when_setting_is_unset() { + let version = std::sync::Mutex::new("1.1".to_string()); + let settings = std::collections::HashMap::new(); + + apply_ocsf_schema_version_setting(&version, &settings); + + assert_eq!(*version.lock().unwrap(), ""); + } + #[test] fn agent_proposals_setting_enables_from_initial_settings_snapshot() { let mut settings = std::collections::HashMap::new(); @@ -6396,6 +6466,7 @@ network_policies: entrypoint_pid: Arc::new(AtomicU32::new(0)), interval_secs: 0, ocsf_enabled: Arc::new(AtomicBool::new(false)), + ocsf_schema_version: Arc::new(std::sync::Mutex::new(String::new())), provider_credentials, provider_readiness, policy_local_ctx: None, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index a24ed8c087..df10d6b32a 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -341,6 +341,7 @@ fn main() -> Result<()> { let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); let _log_push_handle = log_push_state.map(|(_, handle)| handle); let ocsf_enabled = Arc::new(AtomicBool::new(false)); + let ocsf_schema_version = Arc::new(std::sync::Mutex::new(String::new())); let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() @@ -352,7 +353,9 @@ fn main() -> Result<()> { .ok() .map(|roller| { let (writer, guard) = tracing_appender::non_blocking(roller); - let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); + let layer = OcsfJsonlLayer::new(writer) + .with_enabled_flag(ocsf_enabled.clone()) + .with_target_version(ocsf_schema_version.clone()); (layer, guard) }); let (jsonl_layer, jsonl_guard) = @@ -437,6 +440,7 @@ fn main() -> Result<()> { args.ssh_socket_path, args.health_socket_path, ocsf_enabled, + ocsf_schema_version, upstream_proxy_args, backend_descriptor, auth_bundle, From 3107ff1f828f0d5ea960ad5cda2948309fbec8cf Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 22 Sep 2026 17:21:23 +0000 Subject: [PATCH 04/10] ci(security): stage release finding enforcement (#3552) Signed-off-by: Evan Lezar --- .github/workflows/release-tag.yml | 6 +++++- .github/workflows/security-scan.yml | 28 ++++++++++++++++---------- CI.md | 31 +++++++++++++++++++---------- Cargo.lock | 12 +++++------ Cargo.toml | 2 +- architecture/build.md | 8 ++++++-- 6 files changed, 57 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 9bbce87ec9..6c8bc0f31c 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -141,7 +141,11 @@ jobs: uses: ./.github/workflows/security-scan.yml with: candidate_ref: ${{ inputs.tag || github.ref_name }} - allow-high-critical: false + # Temporary observation mode for the pre-existing CodeQL, Trivy, and + # Zizmor backlog. Remove after baseline/delta enforcement is implemented. + # Cargo Deny, Codex Security, and scanner execution errors remain fatal. + fail-on-codex-findings: true + fail-on-static-findings: false images: | ghcr.io/nvidia/openshell/gateway:${{ needs.compute-versions.outputs.source_sha }} ghcr.io/nvidia/openshell/sandbox:${{ needs.compute-versions.outputs.source_sha }} diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index d3a22cd850..36a7fec8c3 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -18,9 +18,13 @@ on: description: Allow a full Codex scan when no previous stable tag exists default: false type: boolean - allow-high-critical: - description: Report HIGH/CRITICAL findings without failing; scanner errors still fail - default: false + fail-on-codex-findings: + description: Fail when Codex Security reports findings + default: true + type: boolean + fail-on-static-findings: + description: Fail when CodeQL, Trivy, or Zizmor reports HIGH/CRITICAL findings + default: true type: boolean images: description: Newline-separated image references for Trivy @@ -44,9 +48,13 @@ on: description: Allow a full Codex scan when no previous stable tag exists default: false type: boolean - allow-high-critical: - description: Report HIGH/CRITICAL findings without failing; scanner errors still fail - default: false + fail-on-codex-findings: + description: Fail when Codex Security reports findings + default: true + type: boolean + fail-on-static-findings: + description: Fail when CodeQL, Trivy, or Zizmor reports HIGH/CRITICAL findings + default: true type: boolean images: description: Newline-separated image references for Trivy @@ -79,7 +87,7 @@ jobs: candidate_ref: ${{ inputs.candidate_ref }} stable_ref: ${{ inputs.stable_ref }} allow_full_bootstrap: ${{ inputs.allow_full_bootstrap }} - fail-on-findings: ${{ !inputs.allow-high-critical }} + fail-on-findings: ${{ inputs.fail-on-codex-findings }} upload_sarif: true secrets: CODEX_SECURITY_API_KEY: ${{ secrets.CODEX_SECURITY_API_KEY }} @@ -92,7 +100,7 @@ jobs: security-events: write with: candidate_ref: ${{ inputs.candidate_ref }} - fail-on-findings: ${{ !inputs.allow-high-critical }} + fail-on-findings: ${{ inputs.fail-on-static-findings }} trivy: name: Trivy @@ -103,7 +111,7 @@ jobs: security-events: write with: candidate_ref: ${{ inputs.candidate_ref }} - fail-on-findings: ${{ !inputs.allow-high-critical }} + fail-on-findings: ${{ inputs.fail-on-static-findings }} severity: HIGH,CRITICAL ignore-unfixed: false images: ${{ inputs.images }} @@ -129,6 +137,6 @@ jobs: security-events: write with: candidate_ref: ${{ inputs.candidate_ref }} - fail-on-findings: ${{ !inputs.allow-high-critical }} + fail-on-findings: ${{ inputs.fail-on-static-findings }} secrets: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} diff --git a/CI.md b/CI.md index f76baf9091..35802a42dc 100644 --- a/CI.md +++ b/CI.md @@ -97,10 +97,18 @@ Actions or call it from another workflow. All applicable children analyze the candidate snapshot. Cargo Deny uses its existing NVIDIA self-hosted runner and CI container. +Tagged releases temporarily run CodeQL, Trivy, and Zizmor findings in +observation mode while the existing backlog is triaged. Cargo Deny advisories, +Codex Security findings, and scanner execution errors still block publication. +The temporary `fail-on-static-findings: false` override must be removed when +baseline/delta enforcement is implemented; it does not waive Cargo Deny, Codex +Security, or scanner execution failures. + ```shell gh workflow run security-scan.yml --ref main \ -f candidate_ref=v0.1.1-pre.1 \ - -F allow-high-critical=true + -F fail-on-codex-findings=false \ + -F fail-on-static-findings=false ``` To integrate it into a larger workflow, run it after the job that pushes the @@ -125,7 +133,8 @@ jobs: ${{ needs.build.outputs.gateway_image }} ${{ needs.build.outputs.sandbox_image }} charts: ${{ needs.build.outputs.chart_ref }} - allow-high-critical: false + fail-on-codex-findings: true + fail-on-static-findings: true secrets: CODEX_SECURITY_API_KEY: ${{ secrets.CODEX_SECURITY_API_KEY }} CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} @@ -140,10 +149,12 @@ publishes SARIF, including Codex results for manual parent runs. Codex keeps its release-train category on `main`; CodeQL, Trivy, and workflow reports publish against the candidate tag and commit. Existing standalone triggers stay active. -The parent fails on HIGH/CRITICAL findings from Codex, CodeQL, Trivy, and Zizmor. -Set `allow-high-critical: true` to report those findings without failing; it -defaults to `false`. Scanner setup, execution, and report publication errors -still fail. Reports are published before the finding threshold is enforced. +The parent fails on Codex findings when `fail-on-codex-findings` is `true` and +on HIGH/CRITICAL findings from CodeQL, Trivy, and Zizmor when +`fail-on-static-findings` is `true`. Both inputs default to `true`. Set either +input to `false` to make only that finding class informational. Scanner setup, +execution, and report publication errors still fail. Reports are published +before the finding threshold is enforced. Codex uses each finding's severity; CodeQL uses the rule's security score (at least 7.0); Trivy uses `HIGH,CRITICAL`, including vulnerabilities without an @@ -151,9 +162,9 @@ upstream fix; Zizmor uses its High severity. Actionlint remains informational. Existing scanner exceptions still apply. Cargo Deny runs only `cargo deny check advisories` in the parent. It keeps its -native failure behavior and configured exceptions, regardless of -`allow-high-critical`; it has no HIGH/CRITICAL filter. Its standalone runs still -check all dependency policies. +native failure behavior and configured exceptions regardless of either finding +threshold input; it has no HIGH/CRITICAL filter. Its standalone runs still check +all dependency policies. Codex scans the cumulative diff from `stable_ref` to `candidate_ref`. Both inputs are tag names, not arbitrary commit SHAs. Omit `stable_ref` to resolve the previous @@ -377,7 +388,7 @@ These workflows run after merge to publish dev/tagged artifacts and verify them. | File | Role | |---|---| | `.github/workflows/release-dev.yml` | Publishes the rolling `dev` build on every push to `main`. Builds gateway, sandbox, and supervisor images and binaries, packages, wheels, and pushes the Helm chart as `oci://ghcr.io/nvidia/openshell/helm-chart:0.0.0-dev` (plus an immutable `0.0.0-dev.` pin). Also dispatchable manually. | -| `.github/workflows/release-tag.yml` | Publishes tagged stable releases and manually dispatched pre-releases. Its automatic tag trigger excludes `-pre.*`. Security Scan gates release publication. | +| `.github/workflows/release-tag.yml` | Publishes tagged stable releases and manually dispatched pre-releases. Its automatic tag trigger excludes `-pre.*`. Security Scan gates release publication on Cargo Deny advisories, Codex Security findings, and scanner execution errors while the static-analysis backlog remains in temporary observation mode. | | `.github/workflows/release-canary.yml` | Smoke-tests published dev artifacts on `macos`, `ubuntu`, `fedora`, and `kubernetes` (kind + Helm) runners. Each job reaches its gateway and creates, exercises, and deletes a sandbox. It runs automatically after `Release Dev` succeeds and supports manual dispatch (`gh workflow run release-canary.yml --ref `). See the `test-release-canary` skill for the playbook and local kind reproduction. | ## Required status contexts diff --git a/Cargo.lock b/Cargo.lock index fe72444b6f..96d6617bb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9018,9 +9018,9 @@ dependencies = [ [[package]] name = "z3" -version = "0.20.2" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c4de445f5c9e3013703a6b8a40c80b4a64c925f0a19e7d0a23a7a9b70e854d" +checksum = "08975512fd325de2a15eede8ed293b2189c745e0770886fae2b30ae229c5b886" dependencies = [ "log", "z3-sys", @@ -9028,18 +9028,18 @@ dependencies = [ [[package]] name = "z3-src" -version = "416.0.2" +version = "501.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2af0c6527de39877cf55cb87f233016573eeeb7cf77afdc1469e4b32faef832" +checksum = "5b72734fb46b47cd6c5a9a1dab4ca1c3c060822d3f34cdcac5d6ae570408b47a" dependencies = [ "cmake", ] [[package]] name = "z3-sys" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18b0a91a13522d21b3414847667de2b2056a721a3edcb5b6ee6858352d58db4" +checksum = "96975839f50a909f6ec19671e5c06d7f0d6fc67287a120e1b010c5f8d91944d6" dependencies = [ "pkg-config", "reqwest 0.12.28", diff --git a/Cargo.toml b/Cargo.toml index f57b22c397..2e7237190f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,7 +144,7 @@ uuid = { version = "1.10", features = ["v4"] } signal-hook = "0.3" # SMT solver (uses system libz3; enable z3/bundled via the prover's bundled-z3 feature for local dev without system z3) -z3 = "0.20" +z3 = "0.21" [workspace.lints.rust] unsafe_code = "warn" diff --git a/architecture/build.md b/architecture/build.md index a75f8ff6ca..0ebb9d2770 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -512,8 +512,12 @@ The tagged release workflow calls the aggregate Security Scan after publishing the candidate's commit-addressed gateway, sandbox, and supervisor images. CodeQL, Trivy, Cargo Deny, and Actionlint/Zizmor run for every release tag; Codex Security also runs for pre-release tags. The release job depends on the aggregate result, -which fails on scanner errors, Cargo Deny advisories, and unexcepted High or -Critical findings. +which fails on scanner errors, Cargo Deny advisories, and Codex Security +findings. CodeQL, Trivy, and Zizmor findings temporarily remain informational +while their existing backlog is triaged. Remove the release workflow's +`fail-on-static-findings: false` override when baseline/delta enforcement is +implemented so new High or Critical findings become blocking without requiring +the historical backlog to reach zero first. `release-auto-tag.yml` runs at 14:00 Europe/Zurich on weekdays (including daylight saving time changes) and supports manual dispatch. Maintainers start weekday From 35e0a68e4aa9878ebc157fe9070357a7cb8b647f Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Tue, 22 Sep 2026 17:57:40 +0000 Subject: [PATCH 05/10] feat(kubernetes): support corporate proxy CA bundle (#3447) * feat(kubernetes): support corporate proxy CA bundle The Kubernetes driver had no way to supply a CA bundle for the corporate egress proxy, so an `https://` proxy with a private CA, or a TLS-intercepting proxy, could not be used. Podman and VM already expose `proxy_ca_bundle`. Add `proxy_ca_bundle` to `[openshell.drivers.kubernetes]` as a path the gateway Pod reads. The gateway stages the PEM into the existing per-generation supervisor bootstrap Secret and passes `--upstream-proxy-ca-bundle` on the supervisor argv. That Secret is already immutable, owner-referenced and garbage-collected, and its volume mounts every key at /.openshell/supervisor with no items filter, so this needs no new object kind, volume, mount, or RBAC verb, and works in shared, managed and operator workspace modes. The bundle is deliberately read from the gateway's filesystem rather than referenced as an object in the sandbox namespace. It becomes a trust anchor for every upstream the sandbox reaches, so it must stay in the gateway's trust domain; the immutable staging Secret also keeps the anchor from changing underneath a running sandbox. Bound the staged bundle at 256 KiB. The shared reader's limit is exactly the apiserver's own Secret limit and the bootstrap Secret carries four other keys, so a bundle between the two would pass gateway startup and then fail every sandbox create with an opaque `data: Too long`. Delegate the URL, no_proxy, connect_by_hostname and ca_bundle rules to the shared validate_upstream_proxy_settings, keeping the Secret-specific credential block local: this driver accepts an explicit `proxy_auth_allow_insecure = false` without credentials, which the shared rules reject. This also fixes the acknowledgement being demanded for an `https://` proxy, where the credential travels inside the verified TLS session. Add auth_setting_label so the inline-credential diagnostic names the Secret keys instead of proxy_auth_file, which this driver rejects as an unknown key. Document that the bundle should carry only the CA that signs the proxy's certificate, or that an intercepting proxy re-signs upstream certificates with. Public roots already reach the sandbox through the supervisor image and its TLS stack, and the bundle is concatenated with that system store into a single boundary control frame, so a full merged trust bundle spends the frame budget on duplicated roots. The frame, not the apiserver Secret limit, is the tighter of the two ceilings in practice; raising the staging bound requires checking it. Closes #3443 Signed-off-by: Philippe Martin * fix(helm): quote proxy CA ConfigMap references Signed-off-by: Philippe Martin --------- Signed-off-by: Philippe Martin --- architecture/sandbox.md | 9 + crates/openshell-core/src/driver_utils.rs | 38 +++- crates/openshell-driver-kubernetes/README.md | 5 +- .../openshell-driver-kubernetes/src/config.rs | 186 ++++++++++++++--- .../openshell-driver-kubernetes/src/driver.rs | 143 +++++++++++++ .../openshell-driver-kubernetes/src/main.rs | 8 + .../src/sandbox_runtime.rs | 191 ++++++++++++++++-- deploy/helm/openshell/README.md | 6 +- .../openshell/templates/_gateway-workload.tpl | 15 ++ .../openshell/templates/gateway-config.yaml | 3 + .../openshell/tests/gateway_config_test.yaml | 19 ++ .../tests/gateway_upstream_proxy_ca_test.yaml | 110 ++++++++++ deploy/helm/openshell/values.yaml | 19 +- docs/reference/gateway-config.mdx | 26 ++- docs/reference/sandbox-compute-drivers.mdx | 25 ++- e2e/rust/tests/kubernetes_corporate_proxy.rs | 62 +++++- e2e/with-kube-gateway.sh | 64 +++++- skills/debug-openshell-cluster/SKILL.md | 32 ++- 18 files changed, 891 insertions(+), 70 deletions(-) create mode 100644 deploy/helm/openshell/tests/gateway_upstream_proxy_ca_test.yaml diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 815fbef0c2..e8412a43f2 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -461,6 +461,15 @@ every intercepted upstream connection failing. The bundle is valid with either an `http://` or `https://` proxy (an intercepting proxy can be reached over plain HTTP) and is fail-closed: an unreadable or certificate-free file is fatal. +How the bundle reaches the supervisor is driver-specific. Drivers that run the +supervisor locally bind-mount the operator's file. The Kubernetes driver cannot: +the supervisor Pod is scheduled remotely, and a trust anchor for every upstream +the sandbox reaches must not be sourced from the workload namespace, where it +would widen to anyone holding write access there. The gateway instead reads the +PEM from its own filesystem and stages it into the per-generation supervisor +bootstrap Secret, which is immutable, so the anchor cannot change underneath a +running sandbox. + Proxy credentials are never embedded in the URL: an inline `user:pass@` is rejected because it would be stored in `gateway.toml` and exposed in container metadata. Operators supply credentials via `proxy_auth_file`; the driver diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 33bff374e5..a5a86fe908 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -561,7 +561,9 @@ fn read_regular_file_bounded(path: &str, max_bytes: u64) -> Result { /// `https_proxy`: the corporate forward proxy URL. @@ -578,6 +580,14 @@ pub struct UpstreamProxySettings<'a> { pub connect_by_hostname: Option, /// `proxy_ca_bundle`: host path to a PEM CA bundle trusted for the proxy. pub ca_bundle: Option<&'a str>, + /// Operator-facing name of this driver's credential setting, used only in + /// diagnostics. `None` means `proxy_auth_file`, the local-driver default. + /// + /// Drivers that deliver the credential by another mechanism set this so an + /// error never points the operator at a key their driver rejects: the + /// Kubernetes driver takes credentials from a Secret and denies unknown + /// config keys, so `proxy_auth_file` would be actively misleading there. + pub auth_setting_label: Option<&'a str>, } /// Validate operator-supplied corporate upstream-proxy settings, fail-closed. @@ -603,9 +613,11 @@ pub fn validate_upstream_proxy_settings( let addr = parse_upstream_proxy_url(url).map_err(|err| match err { UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), UpstreamProxyUrlError::InlineCredentials => { - "https_proxy must not embed credentials in the URL; supply them via \ - proxy_auth_file so they are not stored in config or sandbox metadata" - .to_string() + let auth_setting = settings.auth_setting_label.unwrap_or("proxy_auth_file"); + format!( + "https_proxy must not embed credentials in the URL; supply them via \ + {auth_setting} so they are not stored in config or sandbox metadata" + ) } err => format!("https_proxy {err}"), })?; @@ -1333,6 +1345,24 @@ mod tests { assert!(err.contains("proxy_auth_file"), "{err}"); } + #[test] + fn upstream_proxy_settings_name_the_driver_credential_setting_in_diagnostics() { + // A driver that takes credentials elsewhere must not send the operator + // looking for `proxy_auth_file`; the Kubernetes driver denies unknown + // config keys, so naming it there would be an actively wrong hint. + let err = validate_upstream_proxy_settings(&UpstreamProxySettings { + url: Some("http://u:p@proxy:3128"), + auth_setting_label: Some("proxy_auth_secret_name and proxy_auth_secret_key"), + ..UpstreamProxySettings::default() + }) + .expect_err("inline credentials would be stored in gateway config"); + assert!( + err.contains("proxy_auth_secret_name and proxy_auth_secret_key"), + "{err}" + ); + assert!(!err.contains("proxy_auth_file"), "{err}"); + } + #[test] fn upstream_proxy_settings_reject_an_empty_proxy_url() { let err = validate_upstream_proxy_settings(&proxy_settings(Some(" "))) diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 07f8a0ffe1..16c12d0505 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -80,7 +80,10 @@ CNI enforces ingress and egress `NetworkPolicy` for sandbox namespaces. Each sandbox generation uses two immutable bootstrap Secrets. A trusted init container stages the sandbox bootstrap into memory, and the sandbox removes it before starting untrusted code. The other Secret is mounted only by the -supervisor. The TLS channel binds the namespace, Sandbox CR, workload Pod, +supervisor; when `proxy_ca_bundle` is configured it also carries the operator's +corporate proxy CA bundle, which the gateway reads from its own filesystem so +the anchor stays in the gateway's trust domain rather than the sandbox +namespace. The TLS channel binds the namespace, Sandbox CR, workload Pod, supervisor Pod, and shared network-policy identities. Stop deletes the workload and supervisor Pods. Start rotates both Secrets and creates a new supervisor Pod before releasing a new workload Pod. The shared network fence remains for diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index f29ba79f5b..1574bee36f 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -55,6 +55,19 @@ pub const DEFAULT_GATEWAY_ID: &str = "openshell"; /// Default Kubernetes namespace for sandbox resources. pub const DEFAULT_K8S_NAMESPACE: &str = "openshell"; +/// Hard upper bound on a `proxy_ca_bundle` staged into a bootstrap Secret. +/// +/// The shared reader already bounds the file at +/// [`MAX_UPSTREAM_PROXY_CA_BUNDLE_BYTES`](openshell_core::driver_utils::MAX_UPSTREAM_PROXY_CA_BUNDLE_BYTES), +/// but that bound is exactly the apiserver's own ~1 MiB limit on a Secret, +/// and the bootstrap Secret carries four other keys besides this one. A +/// bundle between the two limits would be accepted on the gateway host and +/// then fail every sandbox create with an opaque apiserver `data: Too long` +/// error, so this driver bounds it well clear of that ceiling and names the +/// operator setting in the error. A bundle holding every corporate trust +/// anchor is a few tens of kilobytes. +pub const MAX_STAGED_PROXY_CA_BUNDLE_BYTES: usize = 256 * 1024; + /// Default Kubernetes `ServiceAccount` assigned to sandbox pods. pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; @@ -217,6 +230,25 @@ pub struct KubernetesComputeConfig { /// Send hostnames rather than validated IPs in CONNECT requests. This is a /// last-resort compatibility mode for hostname-filtering proxy ACLs. pub proxy_connect_by_hostname: Option, + /// Path, on the gateway Pod's filesystem, to a PEM CA bundle trusted for + /// the corporate proxy. + /// + /// A CA certificate is not secret, so the gateway reads this file and + /// stages it into the per-generation supervisor bootstrap Secret, which + /// kubelet mounts read-only in the supervisor Pod. The supervisor trusts + /// it for the TLS handshake with an `https://` proxy and, because a + /// TLS-intercepting proxy re-signs tunneled server certificates with the + /// same CA, folds it into the sandbox trust bundle and upstream + /// verification. + /// + /// The bundle is deliberately read from the gateway's own filesystem + /// rather than referenced as an object in the sandbox namespace: it + /// becomes a trust anchor for every upstream the sandbox reaches, so it + /// must stay in the gateway's trust domain. The staging Secret is + /// immutable, so the anchor cannot change underneath a running sandbox. + /// Only meaningful with `https_proxy` set; the bundle must exist and + /// contain at least one usable trust anchor. + pub proxy_ca_bundle: Option, pub grpc_endpoint: String, pub ssh_socket_path: String, pub client_tls_secret_name: String, @@ -318,6 +350,7 @@ impl Default for KubernetesComputeConfig { proxy_auth_secret_key: None, proxy_auth_allow_insecure: None, proxy_connect_by_hostname: None, + proxy_ca_bundle: None, grpc_endpoint: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), client_tls_secret_name: String::new(), @@ -376,25 +409,46 @@ impl KubernetesComputeConfig { } /// Validate the operator-owned corporate upstream proxy configuration. + /// + /// The URL grammar and the pairing rules for `no_proxy`, + /// `proxy_connect_by_hostname`, and `proxy_ca_bundle` are delegated to + /// [`validate_upstream_proxy_settings`](openshell_core::driver_utils::validate_upstream_proxy_settings) + /// so this driver can never accept a value the supervisor rejects, or + /// drift from the Podman and VM drivers. + /// + /// Credential validation stays local, because this driver takes + /// credentials from a Kubernetes Secret reference rather than the + /// `proxy_auth_file` the shared rules describe. + /// + /// # Errors + /// + /// Returns a message naming the offending key. pub fn validate_upstream_proxy_config(&self) -> Result<(), String> { - use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - - if let Some(url) = &self.https_proxy { - parse_upstream_proxy_url(url).map_err(|err| match err { - UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), - UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), - err => format!("https_proxy {err}"), - })?; - } + use openshell_core::driver_utils::{ + UpstreamProxySettings, parse_upstream_proxy_url, validate_upstream_proxy_settings, + }; - if let Some(list) = self.no_proxy.as_deref() { - if list.trim().is_empty() { - return Err("no_proxy must not be empty when set; omit it instead".to_string()); - } - if self.https_proxy.is_none() { - return Err("no_proxy is set but no https_proxy is configured".to_string()); - } - } + validate_upstream_proxy_settings(&UpstreamProxySettings { + url: self.https_proxy.as_deref(), + no_proxy: self.no_proxy.as_deref(), + connect_by_hostname: self.proxy_connect_by_hostname, + ca_bundle: self.proxy_ca_bundle.as_deref(), + // Credentials are validated below, so the shared auth fields stay + // unset; this label only redirects the inline-credential + // diagnostic away from `proxy_auth_file`, which this driver + // rejects as an unknown key. + auth_setting_label: Some("proxy_auth_secret_name and proxy_auth_secret_key"), + ..UpstreamProxySettings::default() + })?; + + // A credential sent to an `https://` proxy travels inside the verified + // TLS session to that proxy, so the cleartext acknowledgement below is + // only meaningful for an `http://` one. The URL already parsed above. + let proxy_secure = self + .https_proxy + .as_deref() + .and_then(|url| parse_upstream_proxy_url(url).ok()) + .is_some_and(|addr| addr.secure); let secret_name = self.proxy_auth_secret_name.as_deref(); let secret_key = self.proxy_auth_secret_key.as_deref(); @@ -444,8 +498,8 @@ impl KubernetesComputeConfig { .to_string(), ); } - if self.proxy_auth_allow_insecure != Some(true) { - return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); + if !proxy_secure && self.proxy_auth_allow_insecure != Some(true) { + return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, use an https:// proxy, or remove the credential Secret".to_string()); } } _ => { @@ -456,11 +510,6 @@ impl KubernetesComputeConfig { } } - if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { - return Err( - "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), - ); - } Ok(()) } @@ -1166,10 +1215,15 @@ mod tests { proxy_auth_secret_key = "credentials" proxy_auth_allow_insecure = true proxy_connect_by_hostname = true + proxy_ca_bundle = "/etc/openshell-tls/proxy-ca/ca.crt" "#, ) .unwrap(); assert!(cfg.validate_upstream_proxy_config().is_ok()); + assert_eq!( + cfg.proxy_ca_bundle.as_deref(), + Some("/etc/openshell-tls/proxy-ca/ca.crt") + ); assert_eq!( cfg.https_proxy.as_deref(), Some("http://proxy.corp.example:8080") @@ -1317,6 +1371,90 @@ mod tests { assert!(cfg.validate_upstream_proxy_config().is_ok()); } + #[test] + fn upstream_proxy_config_accepts_a_ca_bundle_with_an_https_proxy() { + let cfg = KubernetesComputeConfig { + https_proxy: Some("https://proxy.corp.example:3130".to_string()), + proxy_ca_bundle: Some("/etc/openshell-tls/proxy-ca/ca.crt".to_string()), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + + #[test] + fn upstream_proxy_config_accepts_a_ca_bundle_for_an_intercepting_http_proxy() { + // A proxy reached over plain HTTP still re-signs tunneled upstream + // certificates when it intercepts TLS, so the bundle is meaningful + // without an `https://` proxy URL. + let cfg = KubernetesComputeConfig { + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + proxy_ca_bundle: Some("/etc/openshell-tls/proxy-ca/ca.crt".to_string()), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + + #[test] + fn upstream_proxy_config_rejects_a_ca_bundle_without_a_proxy() { + let cfg = KubernetesComputeConfig { + proxy_ca_bundle: Some("/etc/openshell-tls/proxy-ca/ca.crt".to_string()), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_upstream_proxy_config().unwrap_err(); + assert!(err.contains("proxy_ca_bundle"), "{err}"); + } + + #[test] + fn upstream_proxy_config_rejects_an_empty_ca_bundle() { + // Present-but-empty is a misconfiguration, not "unset": the supervisor + // treats an empty driver-supplied argument as a fatal error. + let cfg = KubernetesComputeConfig { + https_proxy: Some("https://proxy.corp.example:3130".to_string()), + proxy_ca_bundle: Some(" ".to_string()), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_upstream_proxy_config().unwrap_err(); + assert!(err.contains("proxy_ca_bundle"), "{err}"); + } + + #[test] + fn upstream_proxy_config_accepts_credentials_without_acknowledgement_for_an_https_proxy() { + // The credential travels inside the verified TLS session to the proxy, + // so the cleartext acknowledgement is not meaningful here. + let cfg = KubernetesComputeConfig { + https_proxy: Some("https://proxy.corp.example:3130".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + + #[test] + fn upstream_proxy_config_still_requires_acknowledgement_for_an_http_proxy() { + let cfg = KubernetesComputeConfig { + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_upstream_proxy_config().unwrap_err(); + assert!(err.contains("proxy_auth_allow_insecure"), "{err}"); + } + + #[test] + fn upstream_proxy_config_names_the_secret_keys_for_inline_credentials() { + // `proxy_auth_file` is a Podman/VM key that this driver rejects under + // `deny_unknown_fields`, so it must never appear in the diagnostic. + let cfg = KubernetesComputeConfig { + https_proxy: Some("http://user:pass@proxy.corp.example:8080".to_string()), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_upstream_proxy_config().unwrap_err(); + assert!(err.contains("proxy_auth_secret_name"), "{err}"); + assert!(!err.contains("proxy_auth_file"), "{err}"); + } + #[test] fn namespace_for_workspace_shared() { let cfg = KubernetesComputeConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index bd71dd3545..f559a4d0d1 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -171,6 +171,45 @@ impl KubernetesDriverError { } } +/// Read and validate the operator's corporate proxy CA bundle, if configured. +/// +/// Returns the PEM bytes to stage into the supervisor bootstrap Secret, or +/// `None` when `proxy_ca_bundle` is unset. Fail-closed: a path that cannot be +/// read, is not a regular file, or holds no usable trust anchor is an error +/// naming the operator setting, never a silent fall-back to the built-in +/// roots. +/// +/// The read is bounded twice. The shared reader applies +/// `MAX_UPSTREAM_PROXY_CA_BUNDLE_BYTES`, which is exactly the apiserver's own +/// Secret limit; this driver then applies the tighter +/// [`MAX_STAGED_PROXY_CA_BUNDLE_BYTES`] so a bundle between the two cannot +/// turn into an opaque `data: Too long` apiserver error on every create. +async fn read_staged_upstream_proxy_ca_bundle( + path: Option<&str>, +) -> Result>, KubernetesDriverError> { + let Some(path) = path.map(str::to_string) else { + return Ok(None); + }; + let pem = tokio::task::spawn_blocking(move || { + openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file(&path, "proxy_ca_bundle") + }) + .await + .map_err(|error| { + KubernetesDriverError::Message(format!("proxy_ca_bundle read task failed: {error}")) + })? + .map_err(KubernetesDriverError::InvalidArgument)?; + + if pem.len() > crate::config::MAX_STAGED_PROXY_CA_BUNDLE_BYTES { + return Err(KubernetesDriverError::InvalidArgument(format!( + "proxy_ca_bundle is {} bytes, exceeding the {}-byte limit for a bundle staged into a \ + Kubernetes Secret", + pem.len(), + crate::config::MAX_STAGED_PROXY_CA_BUNDLE_BYTES + ))); + } + Ok(Some(pem.into_bytes())) +} + fn is_kube_resource_version_conflict(error: &KubeError) -> bool { matches!(error, KubeError::Api(api) if api.code == 409 && api.reason == "Conflict") } @@ -688,6 +727,13 @@ impl KubernetesComputeDriver { config .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; + // Read the bundle once at startup so an unreadable path or a + // certificate-free file fails the gateway here, rather than as a + // repeated sandbox-create failure. The authoritative read still + // happens per sandbox, so rotating the file needs no restart. + read_staged_upstream_proxy_ca_bundle(config.proxy_ca_bundle.as_deref()) + .await + .map_err(|error| KubernetesDriverError::Precondition(error.to_string()))?; let base_config = match kube::Config::incluster() { Ok(c) => c, Err(_) => kube::Config::infer() @@ -2168,6 +2214,11 @@ impl KubernetesComputeDriver { main_process_spec: &str, log_level: &str, ) -> Result { + // Read once per call so the whole generation stages, mounts, and + // references the same bytes, and so a bundle that became unreadable + // fails before any Service, Pod, or Secret is created. + let upstream_proxy_ca_bundle = + read_staged_upstream_proxy_ca_bundle(self.config.proxy_ca_bundle.as_deref()).await?; let cr_uid = sandbox_cr.metadata.uid.as_deref().ok_or_else(|| { KubernetesDriverError::Message("created Sandbox CR has no UID".to_string()) })?; @@ -2243,6 +2294,7 @@ impl KubernetesComputeDriver { .zip(self.config.proxy_auth_secret_key.as_deref()), self.config.proxy_auth_allow_insecure == Some(true), self.config.proxy_connect_by_hostname == Some(true), + upstream_proxy_ca_bundle.is_some(), self.config.provider_spiffe_enabled().then_some( self.config .provider_spiffe_workload_api_socket_path @@ -2432,6 +2484,7 @@ impl KubernetesComputeDriver { })?, proxy_ca.certificate_pem.into_bytes(), proxy_ca.private_key_pem.into_bytes(), + upstream_proxy_ca_bundle.clone(), OwnerReference { api_version: "v1".to_string(), kind: "Pod".to_string(), @@ -2505,6 +2558,11 @@ impl KubernetesComputeDriver { child_env: std::collections::HashMap, launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, ) -> Result<(), KubernetesDriverError> { + // Read once per call so the whole generation stages, mounts, and + // references the same bytes, and so a bundle that became unreadable + // fails before any Service, Pod, or Secret is created. + let upstream_proxy_ca_bundle = + read_staged_upstream_proxy_ca_bundle(self.config.proxy_ca_bundle.as_deref()).await?; let namespace_uid = Api::::all(self.client.clone()) .get(namespace) .await @@ -2675,6 +2733,7 @@ impl KubernetesComputeDriver { })?, proxy_ca.certificate_pem.into_bytes(), proxy_ca.private_key_pem.into_bytes(), + upstream_proxy_ca_bundle.clone(), OwnerReference { api_version: "v1".to_string(), kind: "Pod".to_string(), @@ -2846,6 +2905,11 @@ impl KubernetesComputeDriver { encoded_authentication: &[u8], encoded_expected_runtime_identity: &str, ) -> Result { + // Read once per call so the whole generation stages, mounts, and + // references the same bytes, and so a bundle that became unreadable + // fails before any Service, Pod, or Secret is created. + let upstream_proxy_ca_bundle = + read_staged_upstream_proxy_ca_bundle(self.config.proxy_ca_bundle.as_deref()).await?; let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( encoded_generation.to_string(), ) @@ -3002,6 +3066,7 @@ impl KubernetesComputeDriver { .zip(self.config.proxy_auth_secret_key.as_deref()), self.config.proxy_auth_allow_insecure == Some(true), self.config.proxy_connect_by_hostname == Some(true), + upstream_proxy_ca_bundle.is_some(), self.config.provider_spiffe_enabled().then_some( self.config .provider_spiffe_workload_api_socket_path @@ -7064,6 +7129,84 @@ mod tests { use prost_types::{Struct, Value, value::Kind}; use std::collections::{BTreeSet, VecDeque}; + /// Write `contents` to a uniquely named temp file and return its path. + /// The caller removes it. + fn write_temp_pem(tag: &str, contents: &str) -> PathBuf { + static COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let unique = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "openshell-proxy-ca-{tag}-{}-{unique}.pem", + std::process::id() + )); + std::fs::write(&path, contents).expect("write temp PEM"); + path + } + + #[tokio::test] + async fn staged_proxy_ca_bundle_is_absent_when_unconfigured() { + assert!( + read_staged_upstream_proxy_ca_bundle(None) + .await + .expect("an unset bundle is not an error") + .is_none() + ); + } + + #[tokio::test] + async fn staged_proxy_ca_bundle_reads_a_valid_bundle() { + let material = generate_proxy_ca_material().expect("generate a CA certificate"); + let path = write_temp_pem("valid", &material.certificate_pem); + let staged = read_staged_upstream_proxy_ca_bundle(path.to_str()) + .await + .expect("a CA certificate is a usable trust anchor") + .expect("bundle staged"); + std::fs::remove_file(&path).ok(); + assert_eq!(staged, material.certificate_pem.into_bytes()); + } + + #[tokio::test] + async fn staged_proxy_ca_bundle_fails_closed_and_names_the_setting() { + // Fail-closed: a bundle the operator pointed at must never silently + // degrade to the built-in roots. Every error names `proxy_ca_bundle` + // so the operator can find the setting. + let missing = std::env::temp_dir().join("openshell-proxy-ca-does-not-exist.pem"); + let err = read_staged_upstream_proxy_ca_bundle(missing.to_str()) + .await + .expect_err("an unreadable path is fatal"); + assert!(err.to_string().contains("proxy_ca_bundle"), "{err}"); + + let empty = write_temp_pem("certificate-free", "not a certificate\n"); + let err = read_staged_upstream_proxy_ca_bundle(empty.to_str()) + .await + .expect_err("a certificate-free bundle is fatal"); + std::fs::remove_file(&empty).ok(); + assert!(err.to_string().contains("proxy_ca_bundle"), "{err}"); + + let directory = std::env::temp_dir(); + let err = read_staged_upstream_proxy_ca_bundle(directory.to_str()) + .await + .expect_err("a non-regular file is fatal"); + assert!(err.to_string().contains("proxy_ca_bundle"), "{err}"); + } + + #[tokio::test] + async fn staged_proxy_ca_bundle_rejects_a_bundle_too_large_for_a_secret() { + // Between this bound and the shared 1 MiB reader lies the apiserver's + // own Secret limit, where the failure would otherwise be an opaque + // `data: Too long` on every sandbox create. + let material = generate_proxy_ca_material().expect("generate a CA certificate"); + let repeats = + crate::config::MAX_STAGED_PROXY_CA_BUNDLE_BYTES / material.certificate_pem.len() + 2; + let path = write_temp_pem("oversized", &material.certificate_pem.repeat(repeats)); + let err = read_staged_upstream_proxy_ca_bundle(path.to_str()) + .await + .expect_err("an oversized bundle would fail at the apiserver"); + std::fs::remove_file(&path).ok(); + let err = err.to_string(); + assert!(err.contains("proxy_ca_bundle"), "{err}"); + assert!(err.contains("Secret"), "{err}"); + } + static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index fbcaf1d5f6..8522e58b40 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -159,6 +159,13 @@ struct Args { #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME", action = ArgAction::SetTrue)] proxy_connect_by_hostname: bool, + /// Path to a PEM CA bundle trusted for the corporate proxy. Required for + /// an `https://` proxy with a private CA, and for a TLS-intercepting proxy + /// that re-signs upstream certificates. Read by this process and staged + /// into each sandbox's supervisor bootstrap Secret. + #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY_CA_BUNDLE")] + proxy_ca_bundle: Option, + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -263,6 +270,7 @@ async fn main() -> Result<()> { proxy_auth_secret_key: args.proxy_auth_secret_key, proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), + proxy_ca_bundle: args.proxy_ca_bundle, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 791cbda3c3..07c0572436 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -32,6 +32,10 @@ pub const BOUNDARY_PRIVATE_KEY: &str = "tls.key"; pub const SUPERVISOR_AUTH_BUNDLE_KEY: &str = "auth.json"; pub const PROXY_CA_CERTIFICATE_KEY: &str = "proxy-ca.crt"; pub const PROXY_CA_PRIVATE_KEY: &str = "proxy-ca.key"; +/// Optional bootstrap-Secret key holding the operator's corporate proxy CA +/// bundle (`proxy_ca_bundle`). Distinct from [`PROXY_CA_CERTIFICATE_KEY`], +/// which is the sandbox's own generated TLS-interception CA. +pub const UPSTREAM_PROXY_CA_BUNDLE_KEY: &str = "upstream-proxy-ca.pem"; pub const SANDBOX_BOOTSTRAP_INPUT_PATH: &str = "/.openshell/bootstrap-input"; pub const BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json"; pub const BOUNDARY_CERTIFICATE_PATH: &str = "/.openshell/state/bootstrap/tls.crt"; @@ -40,6 +44,12 @@ pub const BACKEND_DESCRIPTOR_PATH: &str = "/.openshell/supervisor/runtime-descri pub const SUPERVISOR_AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json"; pub const PROXY_CA_CERTIFICATE_PATH: &str = "/.openshell/supervisor/proxy-ca.crt"; pub const PROXY_CA_PRIVATE_KEY_PATH: &str = "/.openshell/supervisor/proxy-ca.key"; +/// Where [`UPSTREAM_PROXY_CA_BUNDLE_KEY`] lands in the supervisor container. +/// +/// The `bootstrap` volume mounts the whole Secret read-only at +/// `/.openshell/supervisor` with no `items` filter, so the key appears here +/// without a dedicated volume or mount. +pub const UPSTREAM_PROXY_CA_BUNDLE_PATH: &str = "/.openshell/supervisor/upstream-proxy-ca.pem"; pub const CONTROL_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock"; pub const NAMESPACE_WORKLOAD_POLICY_NAME: &str = "openshell-sandbox-workloads"; pub const NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME: &str = "openshell-sandbox-supervisors"; @@ -185,6 +195,7 @@ pub fn supervisor_pod( proxy_auth_secret: Option<(&str, &str)>, proxy_auth_allow_insecure: bool, proxy_connect_by_hostname: bool, + upstream_proxy_ca_bundle_staged: bool, provider_spiffe_socket_path: Option<&str>, owner: OwnerReference, ) -> Result { @@ -297,6 +308,14 @@ pub fn supervisor_pod( if proxy_connect_by_hostname { command.push("--upstream-proxy-connect-by-hostname".to_string()); } + // The bundle rides in the bootstrap Secret already mounted at + // /.openshell/supervisor, so this needs no volume of its own. + if upstream_proxy_ca_bundle_staged { + command.extend([ + "--upstream-proxy-ca-bundle".to_string(), + UPSTREAM_PROXY_CA_BUNDLE_PATH.to_string(), + ]); + } if let Some((secret_name, secret_key)) = proxy_auth_secret { let auth_path = Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH); let mount_path = auth_path @@ -466,8 +485,32 @@ pub fn supervisor_bootstrap_secret( supervisor_auth_bundle: Vec, proxy_ca_certificate: Vec, proxy_ca_private_key: Vec, + upstream_proxy_ca_bundle: Option>, owner: OwnerReference, ) -> Secret { + let mut data = BTreeMap::from([ + ( + BACKEND_DESCRIPTOR_KEY.to_string(), + ByteString(backend_descriptor), + ), + ( + SUPERVISOR_AUTH_BUNDLE_KEY.to_string(), + ByteString(supervisor_auth_bundle), + ), + ( + PROXY_CA_CERTIFICATE_KEY.to_string(), + ByteString(proxy_ca_certificate), + ), + ( + PROXY_CA_PRIVATE_KEY.to_string(), + ByteString(proxy_ca_private_key), + ), + ]); + // Staged only when the operator configured `proxy_ca_bundle`; the + // supervisor keys its behavior off the argument, not the file's presence. + if let Some(bundle) = upstream_proxy_ca_bundle { + data.insert(UPSTREAM_PROXY_CA_BUNDLE_KEY.to_string(), ByteString(bundle)); + } Secret { metadata: ObjectMeta { name: Some(names.supervisor_secret.clone()), @@ -476,24 +519,7 @@ pub fn supervisor_bootstrap_secret( labels: Some(common_labels(sandbox_id, SUPERVISOR_SECRET_COMPONENT)), ..Default::default() }, - data: Some(BTreeMap::from([ - ( - BACKEND_DESCRIPTOR_KEY.to_string(), - ByteString(backend_descriptor), - ), - ( - SUPERVISOR_AUTH_BUNDLE_KEY.to_string(), - ByteString(supervisor_auth_bundle), - ), - ( - PROXY_CA_CERTIFICATE_KEY.to_string(), - ByteString(proxy_ca_certificate), - ), - ( - PROXY_CA_PRIVATE_KEY.to_string(), - ByteString(proxy_ca_private_key), - ), - ])), + data: Some(data), immutable: Some(true), type_: Some("Opaque".to_string()), ..Default::default() @@ -641,6 +667,7 @@ mod tests { None, false, false, + false, None, owner(), ) @@ -792,6 +819,7 @@ mod tests { Vec::new(), Vec::new(), Vec::new(), + None, owner(), ); assert_eq!( @@ -813,6 +841,133 @@ mod tests { BACKEND_DESCRIPTOR_KEY.to_string(), ]) ); + + // With `proxy_ca_bundle` configured the operator bundle rides along in + // the same immutable Secret, so kubelet cannot live-update the trust + // anchor underneath a running sandbox. + let staged = supervisor_bootstrap_secret( + "sandbox", + &names, + "pair", + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + Some(b"-----BEGIN CERTIFICATE-----\n".to_vec()), + owner(), + ); + assert_eq!(staged.immutable, Some(true)); + let staged_data = staged.data.expect("Secret data"); + assert_eq!( + staged_data + .get(UPSTREAM_PROXY_CA_BUNDLE_KEY) + .map(|value| value.0.clone()), + Some(b"-----BEGIN CERTIFICATE-----\n".to_vec()) + ); + // The generated interception CA is a separate key and must not be + // overwritten by the operator bundle. + assert!(staged_data.contains_key(PROXY_CA_CERTIFICATE_KEY)); + } + + /// Render a supervisor Pod configured for an `https://` corporate proxy, + /// varying only whether the operator CA bundle was staged. + fn supervisor_pod_with_staged_proxy_ca(staged: bool) -> Pod { + let names = SandboxRuntimeNames::new("pair"); + supervisor_pod( + "sandbox", + &names, + "pair", + "demo", + "gateway", + "supervisor:latest", + None, + "sandbox-sa", + 1000, + 1000, + &[], + "https://gateway:8080", + "client-tls", + "{}", + "info", + 600, + Some("https://proxy.corp.example:3130"), + None, + None, + false, + false, + staged, + None, + owner(), + ) + .expect("render supervisor Pod") + } + + #[test] + fn supervisor_pod_passes_upstream_proxy_ca_bundle_argument() { + let pod = supervisor_pod_with_staged_proxy_ca(true); + let spec = pod.spec.as_ref().expect("Pod spec"); + let container = &spec.containers[0]; + let command = container.command.as_ref().expect("supervisor command"); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy-ca-bundle", UPSTREAM_PROXY_CA_BUNDLE_PATH]), + "{command:?}" + ); + } + + #[test] + fn supervisor_pod_omits_the_ca_bundle_argument_when_unstaged() { + // The supervisor keys its behavior off the argument, so an unstaged + // bundle must not leave a dangling path on the argv. + let pod = supervisor_pod_with_staged_proxy_ca(false); + let spec = pod.spec.as_ref().expect("Pod spec"); + let command = spec.containers[0] + .command + .as_ref() + .expect("supervisor command"); + assert!( + !command + .iter() + .any(|arg| arg == "--upstream-proxy-ca-bundle"), + "{command:?}" + ); + } + + #[test] + fn staging_the_ca_bundle_adds_no_volume_or_mount() { + // The bundle rides in the bootstrap Secret, whose volume already + // mounts every key at /.openshell/supervisor. A new volume here would + // mean the Secret is being projected with an `items` filter that would + // silently drop the new key. + let staged = supervisor_pod_with_staged_proxy_ca(true); + let bare = supervisor_pod_with_staged_proxy_ca(false); + let volume_names = |pod: &Pod| { + pod.spec + .as_ref() + .expect("Pod spec") + .volumes + .as_ref() + .expect("volumes") + .iter() + .map(|volume| volume.name.clone()) + .collect::>() + }; + let mount_paths = |pod: &Pod| { + pod.spec.as_ref().expect("Pod spec").containers[0] + .volume_mounts + .as_ref() + .expect("volume mounts") + .iter() + .map(|mount| mount.mount_path.clone()) + .collect::>() + }; + assert_eq!(volume_names(&staged), volume_names(&bare)); + assert_eq!(mount_paths(&staged), mount_paths(&bare)); + assert!( + UPSTREAM_PROXY_CA_BUNDLE_PATH.starts_with("/.openshell/supervisor/"), + "the bundle must land inside the bootstrap mount" + ); } #[test] diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 160f543afa..8fc1e4719f 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -340,13 +340,15 @@ discovery endpoint or its TLS CA. | supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. | | supervisor.sandboxRuntime.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. | | tolerations | list | `[]` | Tolerations for the gateway pod. | -| upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | +| upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"caBundle":{"configMapName":"","key":"ca.crt"},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | | upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | | upstreamProxy.authSecret.key | string | `""` | Secret key containing the proxy credential. | | upstreamProxy.authSecret.name | string | `""` | Existing Secret in the sandbox namespace containing a user:pass value. | +| upstreamProxy.caBundle.configMapName | string | `""` | ConfigMap in the release namespace holding the corporate proxy CA bundle. Required for an https:// proxy with a private CA, and for a TLS-intercepting proxy that re-signs upstream certificates. The gateway reads it and stages it into each sandbox's immutable supervisor bootstrap Secret, so the anchor stays in the gateway's trust domain rather than the workload namespace. Supply only the CA that signs your proxy's certificate, or that the proxy re-signs intercepted upstream certificates with. Public roots already come from the supervisor image and its TLS stack, so a full merged trust bundle (for example an OpenShift config.openshift.io/inject-trusted-cabundle ConfigMap) adds hundreds of kilobytes of duplicated roots and can exceed the sandbox boundary's control-frame budget. | +| upstreamProxy.caBundle.key | string | `"ca.crt"` | Key inside that ConfigMap. Change this only to reuse an existing ConfigMap whose key is not ca.crt. | | upstreamProxy.connectByHostname | bool | `false` | Last-resort option for hostname-filtering proxy ACLs. It lets the proxy resolve CONNECT targets. | | upstreamProxy.noProxy | string | `""` | Comma-separated destinations that bypass only the corporate proxy. | -| upstreamProxy.url | string | `""` | HTTP proxy URL in http://host:port form. HTTPS-to-proxy is not supported. | +| upstreamProxy.url | string | `""` | Proxy URL in http://host:port or https://host:port form. An https:// proxy whose certificate is not publicly trusted also needs caBundle below. | | workload.allowMultiReplicaStatefulSet | bool | `false` | Allow replicaCount > 1 while rendering a StatefulSet. Prefer workload.kind=deployment for external database-backed multi-replica gateways; this override exists for operators who explicitly require StatefulSet identity or storage semantics. | | workload.kind | string | `"statefulset"` | Gateway workload controller kind. Use `statefulset` for the default SQLite database, or `deployment` when server.externalDbSecret points at an external database. | | workspaceResources.enabled | bool | `true` | Create the sandbox ServiceAccount, Role, RoleBinding, and NetworkPolicy from this chart. Disable for a gateway-only release. | diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index e8d4616232..67ed46d140 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -174,6 +174,11 @@ spec: mountPath: /etc/openshell-tls/vault-ca readOnly: true {{- end }} + {{- if .Values.upstreamProxy.caBundle.configMapName }} + - name: upstream-proxy-ca + mountPath: /etc/openshell-tls/proxy-ca + readOnly: true + {{- end }} {{- if .Values.server.providerTokenGrants.spiffe.enabled }} - name: spiffe-workload-api mountPath: {{ dir .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} @@ -270,6 +275,16 @@ spec: - key: ca.crt path: ca.crt {{- end }} + {{- if .Values.upstreamProxy.caBundle.configMapName }} + - name: upstream-proxy-ca + configMap: + name: {{ .Values.upstreamProxy.caBundle.configMapName | quote }} + items: + # The mounted filename stays fixed so the rendered proxy_ca_bundle + # path does not depend on the operator's ConfigMap key. + - key: {{ .Values.upstreamProxy.caBundle.key | default "ca.crt" | quote }} + path: ca.crt + {{- end }} {{- if .Values.server.providerTokenGrants.spiffe.enabled }} - name: spiffe-workload-api csi: diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 0652db4b07..da3b8cec39 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -184,6 +184,9 @@ data: {{- if .Values.upstreamProxy.connectByHostname }} proxy_connect_by_hostname = true {{- end }} + {{- if .Values.upstreamProxy.caBundle.configMapName }} + proxy_ca_bundle = "/etc/openshell-tls/proxy-ca/ca.crt" + {{- end }} {{- if .Values.server.providerTokenGrants.spiffe.enabled }} provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 4675cd9b6d..0a0428152e 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -282,6 +282,25 @@ tests: path: data["gateway.toml"] pattern: 'proxy_connect_by_hostname\s*=\s*true' + - it: renders proxy_ca_bundle when a corporate CA ConfigMap is configured + template: templates/gateway-config.yaml + set: + upstreamProxy.url: https://proxy.corp.example:3130 + upstreamProxy.caBundle.configMapName: corporate-proxy-ca + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?proxy_ca_bundle\s*=\s*"/etc/openshell-tls/proxy-ca/ca\.crt"' + + - it: omits proxy_ca_bundle when no corporate CA ConfigMap is configured + template: templates/gateway-config.yaml + set: + upstreamProxy.url: http://proxy.corp.example:8080 + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'proxy_ca_bundle\s*=' + - it: uses the gateway built-in supervisor image by default template: templates/gateway-config.yaml asserts: diff --git a/deploy/helm/openshell/tests/gateway_upstream_proxy_ca_test.yaml b/deploy/helm/openshell/tests/gateway_upstream_proxy_ca_test.yaml new file mode 100644 index 0000000000..5663654fdd --- /dev/null +++ b/deploy/helm/openshell/tests/gateway_upstream_proxy_ca_test.yaml @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: gateway upstream proxy CA volume +templates: + - templates/gateway-config.yaml + - templates/statefulset.yaml + - templates/deployment.yaml +release: + name: openshell + namespace: my-namespace + +tests: + # The gateway workload template is shared by the StatefulSet and the + # Deployment, so both kinds must mount the operator's CA bundle. + - it: mounts the corporate proxy CA into the gateway StatefulSet + template: templates/statefulset.yaml + set: + upstreamProxy.url: https://proxy.corp.example:3130 + upstreamProxy.caBundle.configMapName: corporate-proxy-ca + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: upstream-proxy-ca + mountPath: /etc/openshell-tls/proxy-ca + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: upstream-proxy-ca + configMap: + name: corporate-proxy-ca + items: + - key: ca.crt + path: ca.crt + + - it: mounts the corporate proxy CA into the gateway Deployment + template: templates/deployment.yaml + set: + workload.kind: deployment + server.externalDbSecret: openshell-pg + upstreamProxy.url: https://proxy.corp.example:3130 + upstreamProxy.caBundle.configMapName: corporate-proxy-ca + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: upstream-proxy-ca + mountPath: /etc/openshell-tls/proxy-ca + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: upstream-proxy-ca + configMap: + name: corporate-proxy-ca + items: + - key: ca.crt + path: ca.crt + + # The mounted filename stays ca.crt whatever the operator's ConfigMap key is, + # so the rendered proxy_ca_bundle path never has to vary. + - it: projects a custom ConfigMap key onto the fixed mounted filename + template: templates/statefulset.yaml + set: + upstreamProxy.url: https://proxy.corp.example:3130 + upstreamProxy.caBundle.configMapName: openshift-trusted-ca + upstreamProxy.caBundle.key: ca-bundle.crt + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: upstream-proxy-ca + configMap: + name: openshift-trusted-ca + items: + - key: ca-bundle.crt + path: ca.crt + + # Kubernetes requires both ConfigMap reference fields to be strings. Quote + # numeric-looking operator values in the rendered workload so YAML does not + # decode them as numbers before typed Kubernetes validation. + - it: preserves numeric-looking ConfigMap references as strings + template: templates/statefulset.yaml + set: + upstreamProxy.url: https://proxy.corp.example:3130 + upstreamProxy.caBundle.configMapName: "123" + upstreamProxy.caBundle.key: "456" + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: upstream-proxy-ca + configMap: + name: "123" + items: + - key: "456" + path: ca.crt + + - it: adds no volume when no corporate proxy CA is configured + template: templates/statefulset.yaml + set: + upstreamProxy.url: http://proxy.corp.example:8080 + asserts: + - notContains: + path: spec.template.spec.volumes + content: + name: upstream-proxy-ca + any: true diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index f395b7fa58..786038a8d4 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -56,7 +56,8 @@ supervisor: # -- Operator-owned corporate forward proxy for policy-approved TLS egress # from Kubernetes sandboxes. The workload cannot select or override it. upstreamProxy: - # -- HTTP proxy URL in http://host:port form. HTTPS-to-proxy is not supported. + # -- Proxy URL in http://host:port or https://host:port form. An https:// proxy + # whose certificate is not publicly trusted also needs caBundle below. url: "" # -- Comma-separated destinations that bypass only the corporate proxy. noProxy: "" @@ -69,6 +70,22 @@ upstreamProxy: authAllowInsecure: false # -- Last-resort option for hostname-filtering proxy ACLs. It lets the proxy resolve CONNECT targets. connectByHostname: false + caBundle: + # -- ConfigMap in the release namespace holding the corporate proxy CA bundle. + # Required for an https:// proxy with a private CA, and for a TLS-intercepting + # proxy that re-signs upstream certificates. The gateway reads it and stages it + # into each sandbox's immutable supervisor bootstrap Secret, so the anchor stays + # in the gateway's trust domain rather than the workload namespace. + # Supply only the CA that signs your proxy's certificate, or that the proxy + # re-signs intercepted upstream certificates with. Public roots already come + # from the supervisor image and its TLS stack, so a full merged trust bundle + # (for example an OpenShift config.openshift.io/inject-trusted-cabundle + # ConfigMap) adds hundreds of kilobytes of duplicated roots and can exceed + # the sandbox boundary's control-frame budget. + configMapName: "" + # -- Key inside that ConfigMap. Change this only to reuse an existing + # ConfigMap whose key is not ca.crt. + key: ca.crt # -- Image pull secrets attached to gateway and helper pods. imagePullSecrets: [] diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 4acd6808b1..a5c322be89 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -583,10 +583,11 @@ sandbox_runtime_image_pull_policy = "if_not_present" # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "if_not_present" -# Optional corporate HTTP forward proxy for policy-approved TLS egress. The -# sandbox workload cannot select or override these settings. Only http:// proxy -# endpoints and TLS CONNECT traffic are supported; plain HTTP egress remains -# direct. `no_proxy` bypasses only the corporate proxy, never OpenShell policy. +# Optional corporate forward proxy for policy-approved TLS egress. The sandbox +# workload cannot select or override these settings. http:// and https:// proxy +# endpoints are supported, for TLS CONNECT traffic only; plain HTTP egress +# remains direct. `no_proxy` bypasses only the corporate proxy, never OpenShell +# policy. # https_proxy = "http://proxy.corp.example:8080" # no_proxy = ".svc,.svc.cluster.local,10.96.0.0/12,10.244.0.0/16" # Proxy credentials must be an existing Secret in the sandbox namespace. The @@ -601,12 +602,25 @@ supervisor_image_pull_policy = "if_not_present" # content is validated fail-closed by the supervisor at startup and never # falls back to direct egress. # Proxy credentials mount only in the separately scheduled supervisor Pod. -# Required with a credential Secret: Basic authentication to an http:// proxy -# is cleartext on the connection to that proxy. +# Required with a credential Secret when the proxy URL is http://, because +# Basic authentication is then cleartext on the connection to that proxy. An +# https:// proxy carries the credential inside the verified TLS session and +# needs no acknowledgement. # proxy_auth_allow_insecure = true # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true +# CA bundle trusted for the corporate proxy, as a path on the gateway Pod's +# filesystem. Needed for an https:// proxy whose certificate is not publicly +# trusted, and for a TLS-intercepting proxy that re-signs upstream +# certificates. The gateway reads the file and stages it into each sandbox's +# immutable supervisor bootstrap Secret, so the anchor stays in the gateway's +# trust domain and cannot change underneath a running sandbox. Helm mounts it +# from `upstreamProxy.caBundle.configMapName` and renders this path. Supply +# only the proxy's own CA: public roots already come from the supervisor image +# and its TLS stack, so a full merged trust bundle wastes the sandbox +# boundary's control-frame budget on duplicated roots. +# proxy_ca_bundle = "/etc/openshell-tls/proxy-ca/ca.crt" # Required in raw gateway TOML because `namespace` identifies sandbox # placement, not the gateway Service. Helm renders this from the release's # gateway Service name and namespace. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 3291e48d4c..2151c1a69b 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -437,16 +437,37 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | | `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | -| `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | +| `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` or `https://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | | `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. | | `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. The Secret mounts only in the supervisor Pod. | | `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. | -| `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret. | +| `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret and an `http://` proxy; an `https://` proxy carries the credential inside the verified TLS session and needs no acknowledgement. | | `proxy_connect_by_hostname` | `upstreamProxy.connectByHostname` | Send hostnames rather than validated IPs in CONNECT requests. Use only when proxy ACLs require hostname targets. | +| `proxy_ca_bundle` | `upstreamProxy.caBundle.configMapName` / `upstreamProxy.caBundle.key` | Trust a PEM CA bundle for the corporate proxy. Required for an `https://` proxy with a private CA, and for a TLS-intercepting proxy that re-signs upstream certificates. Helm mounts the ConfigMap into the gateway Pod; the gateway stages the bundle into each sandbox's immutable supervisor bootstrap Secret. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | +`proxy_ca_bundle` needs only the CA that signs the proxy's certificate, or that +a TLS-intercepting proxy re-signs upstream certificates with. Public roots +already come from the supervisor image and its TLS stack, so supplying a full +merged trust bundle adds hundreds of kilobytes of duplicated roots for no +benefit. + +On OpenShift, copy the cluster proxy's trusted-CA ConfigMap into the gateway's +release namespace rather than pointing at an injected trusted-CA bundle. A copy +is required in any case, because ConfigMaps cannot be referenced across +namespaces: + +```shell +CORP_CA=$(oc get proxy/cluster -o jsonpath='{.spec.trustedCA.name}') +oc -n openshift-config get cm "$CORP_CA" -o jsonpath='{.data.ca-bundle\.crt}' > corp-ca.pem +oc -n openshell create configmap corporate-proxy-ca --from-file=ca.crt=corp-ca.pem +``` + +Then set `upstreamProxy.caBundle.configMapName` to `corporate-proxy-ca` and +leave `upstreamProxy.caBundle.key` at its `ca.crt` default. + Managed-mode Secret copying requires the gateway ServiceAccount to create Secrets. Kubernetes RBAC cannot restrict Secret `create` by resource name, so the Helm chart grants cluster-wide Secret `create`; Secret `get` and `patch` diff --git a/e2e/rust/tests/kubernetes_corporate_proxy.rs b/e2e/rust/tests/kubernetes_corporate_proxy.rs index 3fc1c12a30..0dc586c53c 100644 --- a/e2e/rust/tests/kubernetes_corporate_proxy.rs +++ b/e2e/rust/tests/kubernetes_corporate_proxy.rs @@ -10,6 +10,13 @@ //! and an HTTPS upstream on the host visible to sandbox pods, then proves that //! a permitted request uses authenticated CONNECT while a policy-denied port //! never reaches the proxy. +//! +//! The `https-ca` mode additionally terminates TLS on the proxy listener with +//! a leaf signed by a private CA that the wrapper publishes as a ConfigMap in +//! the gateway's release namespace. It covers the whole staging chain — +//! gateway read, bootstrap-Secret key, supervisor mount, `--upstream-proxy-ca-bundle` +//! argument — because the CONNECT handshake cannot succeed unless every link +//! works. use std::io::Write as _; @@ -23,9 +30,29 @@ const USER: &str = "proxyuser"; const PASS: &str = "proxypass"; const MARKER: &str = "kubernetes-corporate-proxy-upstream"; -fn proxy_script() -> String { +/// Python source for the forward proxy. +/// +/// With `tls`, the listener is wrapped in TLS using the wrapper-supplied leaf, +/// so the supervisor must complete a verified handshake against the operator +/// CA bundle before it can issue CONNECT. +fn proxy_script(tls: Option<(&str, &str)>) -> String { + let tls_setup = match tls { + Some((cert, key)) => format!( + r#" +import ssl, tempfile, os +_d=tempfile.mkdtemp() +_cert=os.path.join(_d,'proxy.crt'); _key=os.path.join(_d,'proxy.key') +open(_cert,'w').write({cert:?}) +open(_key,'w').write({key:?}) +_ctx=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER); _ctx.load_cert_chain(_cert,_key) +def wrap(sock): + return _ctx.wrap_socket(sock, server_side=True) +"# + ), + None => "\ndef wrap(sock):\n return sock\n".to_string(), + }; format!( - r#" + r#"{tls_setup} import base64, select, socket, threading expected = 'Basic ' + base64.b64encode(b'{USER}:{PASS}').decode() def head(c): @@ -68,7 +95,11 @@ def handle(c): s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1); s.bind(('0.0.0.0',{FIXTURE_PORT})); s.listen(32) print('proxy-listening',flush=True) while True: - c,_=s.accept(); threading.Thread(target=handle,args=(c,),daemon=True).start() + c,_=s.accept() + try: c=wrap(c) + except OSError as e: + print('handshake-failed %s' % e, flush=True); c.close(); continue + threading.Thread(target=handle,args=(c,),daemon=True).start() "# ) } @@ -151,8 +182,22 @@ async fn kubernetes_corporate_proxy_uses_secret_and_never_falls_back() { ); return; } + let proxy_tls = if mode == "https-ca" { + let cert = std::env::var("OPENSHELL_E2E_CORPORATE_PROXY_TLS_CERT") + .expect("https-ca mode requires the proxy leaf certificate from the wrapper"); + let key = std::env::var("OPENSHELL_E2E_CORPORATE_PROXY_TLS_KEY") + .expect("https-ca mode requires the proxy leaf key from the wrapper"); + Some((cert, key)) + } else { + None + }; + let proxy_source = proxy_script( + proxy_tls + .as_ref() + .map(|(cert, key)| (cert.as_str(), key.as_str())), + ); let proxy = - HostSupportContainer::start_python_on_host_port(&proxy_script(), FIXTURE_PORT, proxy_port) + HostSupportContainer::start_python_on_host_port(&proxy_source, FIXTURE_PORT, proxy_port) .await .expect("start authenticated forward proxy"); let upstream = if mode == "no-proxy" { @@ -207,6 +252,15 @@ async fn kubernetes_corporate_proxy_uses_secret_and_never_falls_back() { logs.contains("auth=ok"), "Secret credentials must reach proxy:\n{logs}" ); + if mode == "https-ca" { + // A staging failure anywhere in the chain (gateway read, Secret + // key, mount, argv) shows up here: the supervisor cannot verify + // the listener leaf, so no CONNECT is ever sent. + assert!( + !logs.contains("handshake-failed"), + "supervisor must verify the proxy leaf against the staged CA bundle:\n{logs}" + ); + } assert!( !logs.contains("auth=fail"), "proxy must not receive missing credentials:\n{logs}" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 36c321f071..b9c195b050 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -117,6 +117,8 @@ VAULT_DNS_ALIAS="${VAULT_RELEASE_NAME}-0" VAULT_CA_FILE="${WORKDIR}/openbao-ca.crt" CORPORATE_PROXY_FIXTURE_DEPLOYED=0 CORPORATE_PROXY_FIXTURE_SECRET="openshell-e2e-proxy-auth" +CORPORATE_PROXY_FIXTURE_CA_CONFIGMAP="openshell-e2e-proxy-ca" +CORPORATE_PROXY_CA_FIXTURE_DEPLOYED=0 OPENSHIFT_DETECTED=0 OPENSHIFT_SANDBOX_SCC_GRANTED=0 OPENSHIFT_POSTGRES_SCC_GRANTED=0 @@ -520,6 +522,11 @@ cleanup() { --ignore-not-found >/dev/null 2>&1 || true fi + if [ "${CORPORATE_PROXY_CA_FIXTURE_DEPLOYED}" = "1" ]; then + kctl -n "${NAMESPACE}" delete configmap "${CORPORATE_PROXY_FIXTURE_CA_CONFIGMAP}" \ + --ignore-not-found >/dev/null 2>&1 || true + fi + if [ "${OPENSHIFT_SANDBOX_SCC_GRANTED}" = "1" ]; then oc adm policy remove-scc-from-user privileged \ --context "${KUBE_CONTEXT}" \ @@ -1196,9 +1203,15 @@ if [ "${OPENSHELL_E2E_KUBE_CORPORATE_PROXY:-0}" = "1" ]; then export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-45}" fi CORPORATE_PROXY_VALUES="${WORKDIR}/corporate-proxy-values.yaml" + # `https-ca` terminates TLS on the proxy listener itself, so the supervisor + # must trust the operator CA bundle before it can even issue CONNECT. + CORPORATE_PROXY_SCHEME="http" + if [ "${CORPORATE_PROXY_MODE}" = "https-ca" ]; then + CORPORATE_PROXY_SCHEME="https" + fi cat >"${CORPORATE_PROXY_VALUES}" </dev/null 2>&1 + openssl req -newkey rsa:2048 -nodes \ + -keyout "${CORPORATE_PROXY_TLS_DIR}/proxy.key" \ + -out "${CORPORATE_PROXY_TLS_DIR}/proxy.csr" \ + -subj "/CN=host.openshell.internal" >/dev/null 2>&1 + cat >"${CORPORATE_PROXY_TLS_DIR}/leaf.ext" </dev/null 2>&1 + + # The CA ConfigMap belongs to the gateway release namespace: the gateway + # reads it and stages the bundle per sandbox, so it is never sourced + # from a workload namespace. + kctl -n "${NAMESPACE}" create configmap "${CORPORATE_PROXY_FIXTURE_CA_CONFIGMAP}" \ + --from-file=ca.crt="${CORPORATE_PROXY_TLS_DIR}/ca.crt" \ + --dry-run=client -o yaml | kctl apply -f - + CORPORATE_PROXY_CA_FIXTURE_DEPLOYED=1 + cat >>"${CORPORATE_PROXY_VALUES}" <&2; exit 2 ;; esac diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 67fb2288af..258c1ebde7 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -757,15 +757,32 @@ startup (`validate_upstream_proxy_config`) rather than silently reverting to a direct connection. Confirm the rendered configuration first: ```bash -kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -E 'https_proxy|no_proxy|proxy_auth_secret_(name|key)|proxy_auth_allow_insecure|proxy_connect_by_hostname' -helm -n openshell get values openshell | grep -A8 upstreamProxy +kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -E 'https_proxy|no_proxy|proxy_auth_secret_(name|key)|proxy_auth_allow_insecure|proxy_connect_by_hostname|proxy_ca_bundle' +helm -n openshell get values openshell | grep -A12 upstreamProxy ``` -Only `http://host:port` forward proxies are supported; `https://` proxy URLs and -plain-HTTP egress are out of scope and rejected. The credential Secret named by -`proxy_auth_secret_name` must exist in the sandbox namespace with the key named -by `proxy_auth_secret_key`, and Kubernetes will not create keys longer than 253 -bytes or named `.`/`..`. +Both `http://host:port` and `https://host:port` forward proxies are supported; +plain-HTTP egress is out of scope and always dials directly. The credential +Secret named by `proxy_auth_secret_name` must exist in the sandbox namespace +with the key named by `proxy_auth_secret_key`, and Kubernetes will not create +keys longer than 253 bytes or named `.`/`..`. + +An `https://` proxy with a private CA, or a TLS-intercepting proxy, also needs +`upstreamProxy.caBundle.configMapName`. That ConfigMap lives in the **gateway's** +release namespace, not the sandbox namespace, and the gateway reads it and +stages the bundle into each sandbox's supervisor bootstrap Secret. A missing +ConfigMap leaves the gateway Pod in `ContainerCreating`, like `oidc-ca` and +`vault-ca`: + +```bash +kubectl -n openshell get configmap -o jsonpath='{.data}' >/dev/null && echo "CA ConfigMap present" +kubectl -n openshell describe pod -l app.kubernetes.io/name=openshell | grep -A5 'ContainerCreating\|MountVolume' +kubectl -n get secret -o jsonpath='{.data.upstream-proxy-ca\.pem}' | head -c 20 +``` + +An empty last command with `proxy_ca_bundle` set in the rendered TOML means the +bundle never reached the sandbox; check the gateway logs for a `proxy_ca_bundle` +error, which fails closed at startup rather than falling back to direct egress. The proxy arguments and credential mount belong only to the separate supervisor Pod. The workload Pod must never receive them. The credential is projected @@ -776,6 +793,7 @@ appear in environment variables, annotations, or command arguments. ```bash kubectl -n get secret -o jsonpath='{.data}' >/dev/null && echo "secret present" kubectl -n get pod -o jsonpath='{.spec.containers[0].command}' | grep -- '--upstream-' +kubectl -n get pod -o jsonpath='{.spec.containers[0].command}' | grep -- '--upstream-proxy-ca-bundle' kubectl -n get pod -o jsonpath='{.spec.containers[0].volumeMounts}' | grep upstream-proxy-auth kubectl -n get events --sort-by=.lastTimestamp | grep -Ei 'secret|MountVolume' | tail -n 20 ``` From 718dba34304421af9d5e3ac4f9fea148ba08e0fb Mon Sep 17 00:00:00 2001 From: Yuedong Wu Date: Tue, 22 Sep 2026 17:58:19 +0000 Subject: [PATCH 06/10] fix(policy)!: reject removed tls endpoint values (#3414) Signed-off-by: Yuedong Wu --- architecture/security-policy.md | 7 +- crates/openshell-cli/src/commands/provider.rs | 10 +- .../tests/provider_commands_integration.rs | 1 - crates/openshell-policy/src/l7_validate.rs | 30 ++++-- crates/openshell-policy/src/merge.rs | 26 +---- crates/openshell-prover/src/containment.rs | 27 ++--- crates/openshell-providers/src/profiles.rs | 5 - crates/openshell-server/src/grpc/policy.rs | 4 - .../src/l7/mod.rs | 100 +++++++----------- .../src/l7/rest.rs | 10 +- .../openshell-supervisor-network/src/opa.rs | 3 - .../src/policy_local.rs | 31 +++++- crates/openshell-tui/src/lib.rs | 7 -- docs/about/release-notes.mdx | 12 +++ docs/reference/policy-schema.mdx | 2 +- examples/sandbox-policy-quickstart/README.md | 3 +- proto/sandbox.proto | 4 +- providers/aws-s3.yaml | 6 -- sdk/go/openshell/v1/types/network_policy.go | 4 +- sdk/go/openshell/v1/types_reexport.go | 4 +- sdk/go/proto/sandboxv1/sandbox.pb.go | 4 +- skills/generate-sandbox-policy/SKILL.md | 15 +-- skills/generate-sandbox-policy/examples.md | 9 +- 23 files changed, 145 insertions(+), 179 deletions(-) diff --git a/architecture/security-policy.md b/architecture/security-policy.md index 5164ec4233..98946df833 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -242,8 +242,11 @@ higher specificity rank deterministically overrides broader request-processing metadata. Equally specific overlapping endpoints must agree. Endpoint `tls`, `enforcement`, and `access` use protobuf enums and retain their -named YAML spellings. `protocol` remains a string so the supported protocol set -can evolve, but every ingress validates it before persistence or activation. +named YAML spellings. `tls` admits only an omitted value, meaning auto-detect +and terminate for inspection, or `skip`; every other value, including the +removed `terminate` and `passthrough` spellings, is rejected. `protocol` +remains a string so the supported protocol set can evolve, but every ingress +validates it before persistence or activation. The supervisor also refuses unknown enum numbers and protocol values defensively; an unrecognized enforcement value never falls back to audit. diff --git a/crates/openshell-cli/src/commands/provider.rs b/crates/openshell-cli/src/commands/provider.rs index 6f29ab8443..d7b3e8fcf1 100644 --- a/crates/openshell-cli/src/commands/provider.rs +++ b/crates/openshell-cli/src/commands/provider.rs @@ -2909,13 +2909,7 @@ binaries: [/usr/bin/curl] ) .expect("valid profile fixture"); let mut descriptions = Vec::new(); - for (tls, allow_uninspected_credentials) in [ - ("", false), - ("", true), - ("skip", true), - ("terminate", false), - ("passthrough", false), - ] { + for (tls, allow_uninspected_credentials) in [("", false), ("", true), ("skip", true)] { let mut variant = profile.clone(); variant.endpoints[0].tls = tls.to_string(); variant.endpoints[0].allow_uninspected_credentials = allow_uninspected_credentials; @@ -2947,8 +2941,6 @@ binaries: [/usr/bin/curl] .contains("TLS: skip (raw tunnel; no L7 inspection or credential rewrite)\n") ); assert!(descriptions[2].contains("Allow uninspected credentials: true\n")); - assert!(descriptions[3].contains("TLS: terminate (automatic TLS detection)\n")); - assert!(descriptions[4].contains("TLS: passthrough (automatic TLS detection)\n")); } #[test] diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs index ae1e7363f6..9d06a02035 100644 --- a/crates/openshell-cli/tests/provider_commands_integration.rs +++ b/crates/openshell-cli/tests/provider_commands_integration.rs @@ -4863,7 +4863,6 @@ endpoints: - host: api.advanced.example ports: [443, 8443] protocol: rest - tls: terminate enforcement: enforce rules: - allow: diff --git a/crates/openshell-policy/src/l7_validate.rs b/crates/openshell-policy/src/l7_validate.rs index 112a721978..ac995fa65b 100644 --- a/crates/openshell-policy/src/l7_validate.rs +++ b/crates/openshell-policy/src/l7_validate.rs @@ -68,10 +68,17 @@ pub fn network_access_preset_to_str(value: i32) -> Option<&'static str> { } } +fn unknown_tls_value(tls: &str) -> Option { + (!matches!(tls, "" | "skip")).then(|| { + format!("unknown tls value '{tls}'; omit the field to keep automatic TLS termination") + }) +} + pub fn validate_endpoint_mode_values(tls: i32, enforcement: i32, access: i32) -> Vec { let mut errors = Vec::new(); - if network_tls_mode_to_str(tls).is_none() { - errors.push(format!("unknown tls enum value {tls}")); + match network_tls_mode_to_str(tls) { + Some(value) => errors.extend(unknown_tls_value(value)), + None => errors.push(format!("unknown tls enum value {tls}")), } if network_enforcement_mode_to_str(enforcement).is_none() { errors.push(format!("unknown enforcement enum value {enforcement}")); @@ -174,7 +181,7 @@ mod agent_transport_tests { #[test] fn agent_cannot_request_native_tcp_or_skip_tls_inspection() { assert!(agent_authored_transport_rejection("tcp", "").is_some()); - assert!(agent_authored_transport_rejection("TCP", "terminate").is_some()); + assert!(agent_authored_transport_rejection("TCP", "").is_some()); assert!(agent_authored_transport_rejection("", "skip").is_some()); assert!(agent_authored_transport_rejection("rest", "SKIP").is_some()); } @@ -185,11 +192,7 @@ mod agent_transport_tests { pub fn validate_endpoint_modes(tls: &str, enforcement: &str, access: &str) -> Vec { let mut errors = Vec::new(); - if !matches!(tls, "" | "skip" | "terminate" | "passthrough") { - errors.push(format!( - "unknown tls value '{tls}' (expected skip, terminate, or passthrough)" - )); - } + errors.extend(unknown_tls_value(tls)); if !matches!(enforcement, "" | "enforce" | "audit") { errors.push(format!( "unknown enforcement value '{enforcement}' (expected enforce or audit)" @@ -357,9 +360,18 @@ mod tests { assert!(errors[2].contains("unknown access value 'read-wirte'")); } + #[test] + fn endpoint_mode_values_reject_removed_tls_enums() { + for legacy in [2, 3] { + let errors = validate_endpoint_mode_values(legacy, 0, 0); + assert_eq!(errors.len(), 1, "tls: {legacy}"); + assert!(errors[0].contains("unknown tls value")); + } + } + #[test] fn endpoint_modes_accept_documented_values_and_defaults() { - for tls in ["", "skip", "terminate", "passthrough"] { + for tls in ["", "skip"] { for enforcement in ["", "enforce", "audit"] { for access in ["", "read-only", "read-write", "full"] { assert!(validate_endpoint_modes(tls, enforcement, access).is_empty()); diff --git a/crates/openshell-policy/src/merge.rs b/crates/openshell-policy/src/merge.rs index e64c77d238..2f81b18f3d 100644 --- a/crates/openshell-policy/src/merge.rs +++ b/crates/openshell-policy/src/merge.rs @@ -833,9 +833,7 @@ fn endpoint_attributes_cover(loaded: &NetworkEndpoint, proposed: &NetworkEndpoin if !proposed.protocol.is_empty() && !protocols_match(&loaded.protocol, &proposed.protocol) { return false; } - if proposed.tls != NetworkTlsMode::Unspecified as i32 - && effective_tls(loaded.tls) != effective_tls(proposed.tls) - { + if proposed.tls != NetworkTlsMode::Unspecified as i32 && loaded.tls != proposed.tls { return false; } if proposed.enforcement != NetworkEnforcementMode::Unspecified as i32 @@ -940,19 +938,6 @@ fn protocols_match(left: &str, right: &str) -> bool { } } -#[allow(deprecated)] -fn effective_tls(value: i32) -> i32 { - match value { - value - if value == NetworkTlsMode::Terminate as i32 - || value == NetworkTlsMode::Passthrough as i32 => - { - NetworkTlsMode::Unspecified as i32 - } - value => value, - } -} - fn effective_enforcement(value: i32) -> i32 { if value == NetworkEnforcementMode::Unspecified as i32 { NetworkEnforcementMode::Audit as i32 @@ -3783,7 +3768,6 @@ mod tests { assert!(!policy_covers_rule(&loaded, &different_body)); let mut explicit_defaults = loaded_endpoint; - explicit_defaults.tls = 3; // deprecated passthrough compatibility value explicit_defaults.enforcement = NetworkEnforcementMode::Audit as i32; let runtime_defaults = rule_with_authorizations( "proposed", @@ -3792,14 +3776,6 @@ mod tests { ); assert!(policy_covers_rule(&loaded, &runtime_defaults)); - explicit_defaults.tls = 2; // deprecated terminate compatibility value - let legacy_terminate = rule_with_authorizations( - "proposed", - vec![explicit_defaults.clone()], - &["/usr/bin/client"], - ); - assert!(policy_covers_rule(&loaded, &legacy_terminate)); - explicit_defaults.tls = NetworkTlsMode::Skip as i32; let skip_tls = rule_with_authorizations( "proposed", diff --git a/crates/openshell-prover/src/containment.rs b/crates/openshell-prover/src/containment.rs index 86397f754c..9a9c9cb72a 100644 --- a/crates/openshell-prover/src/containment.rs +++ b/crates/openshell-prover/src/containment.rs @@ -1596,7 +1596,7 @@ fn validate_supported_endpoint_extensions( context: &str, endpoint: &Endpoint, ) -> Result<(), UnsupportedFeature> { - if !matches!(endpoint.tls.as_str(), "" | "terminate" | "passthrough") + if !endpoint.tls.is_empty() || endpoint.allow_encoded_slash || endpoint.websocket_credential_rewrite || endpoint.request_body_credential_rewrite @@ -3184,17 +3184,20 @@ network_policies: } #[test] - fn deprecated_tls_spelling_does_not_change_authority() { - let boundary = parse( - "version: 1\nnetwork_policies:\n n:\n endpoints:\n - host: api.example.com\n port: 443\n protocol: rest\n tls: terminate\n enforcement: enforce\n access: read-only\n binaries: [{ path: /usr/bin/curl }]\n", - ); - let candidate = parse( - "version: 1\nnetwork_policies:\n n:\n endpoints:\n - host: api.example.com\n port: 443\n protocol: rest\n enforcement: enforce\n rules:\n - allow: { method: GET, path: '/v1/**' }\n binaries: [{ path: /usr/bin/curl }]\n", - ); - assert!(matches!( - check_within_boundary(&boundary, &candidate, options()), - CheckResult::Within(_) - )); + fn removed_tls_spelling_is_outside_the_authority_model() { + for tls in ["terminate", "passthrough"] { + let policy = parse(&format!( + "version: 1\nnetwork_policies:\n n:\n endpoints:\n - host: api.example.com\n port: 443\n protocol: rest\n tls: {tls}\n enforcement: enforce\n access: read-only\n binaries: [{{ path: /usr/bin/curl }}]\n" + )); + assert!( + matches!( + check_within_boundary(&policy, &policy, options()), + CheckResult::Unsupported(ref evidence) + if evidence.reason().contains("outside the initial model") + ), + "tls: {tls}" + ); + } } #[test] diff --git a/crates/openshell-providers/src/profiles.rs b/crates/openshell-providers/src/profiles.rs index e5cd698019..01991d712e 100644 --- a/crates/openshell-providers/src/profiles.rs +++ b/crates/openshell-providers/src/profiles.rs @@ -5516,7 +5516,6 @@ endpoints: - host: api.example.com ports: [443, 8443] protocol: rest - tls: terminate enforcement: enforce rules: - allow: @@ -5563,10 +5562,6 @@ binaries: let rest_ep = &proto.endpoints[1]; assert_eq!(rest_ep.port, 0); assert_eq!(rest_ep.ports, vec![443, 8443]); - assert_eq!( - rest_ep.tls, - openshell_core::proto::NetworkTlsMode::Terminate as i32 - ); assert_eq!(rest_ep.allowed_ips, vec!["10.0.0.0/24"]); assert!(rest_ep.allow_encoded_slash); assert!(rest_ep.allow_uninspected_credentials); diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 3324813ad6..27fcf946da 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -10019,14 +10019,12 @@ mod tests { } } - #[allow(deprecated)] fn l7_scope_policy() -> ProtoSandboxPolicy { let endpoint = NetworkEndpoint { host: "api.example.com".to_string(), port: 443, ports: vec![443, 8443], protocol: "rest".to_string(), - tls: openshell_core::proto::NetworkTlsMode::Terminate as i32, access: openshell_core::proto::NetworkAccessPreset::ReadOnly as i32, ..Default::default() }; @@ -10982,7 +10980,6 @@ mod tests { let mut policy = test_policy_with_rule("aws", host); let endpoint = &mut policy.network_policies.get_mut("aws").unwrap().endpoints[0]; endpoint.protocol = "rest".to_string(); - endpoint.tls = 2; endpoint.access = openshell_core::proto::NetworkAccessPreset::Full as i32; endpoint.credential_signing = "sigv4".to_string(); endpoint.signing_service = "s3".to_string(); @@ -13047,7 +13044,6 @@ mod tests { .endpoints[0]; bound_endpoint.protocol = "rest".to_string(); bound_endpoint.access = openshell_core::proto::NetworkAccessPreset::Full as i32; - bound_endpoint.tls = 2; openshell_policy::ensure_sandbox_process_identity(&mut policy); state .store diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index 39d4f11cfc..89c580ed8a 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -307,32 +307,6 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { let tls = match tls_value.as_str() { "skip" => TlsMode::Skip, - "terminate" => { - let event = openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Other, "deprecated") - .message( - "'tls: terminate' is deprecated; TLS termination is now automatic. \ - Use 'tls: skip' to explicitly disable. This field will be removed in a future version.", - ) - .build(); - openshell_ocsf::ocsf_emit!(event); - TlsMode::Auto - } - "passthrough" => { - let event = openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Other, "deprecated") - .message( - "'tls: passthrough' is deprecated; TLS termination is now automatic. \ - Use 'tls: skip' to explicitly disable. This field will be removed in a future version.", - ) - .build(); - openshell_ocsf::ocsf_emit!(event); - TlsMode::Auto - } "" => TlsMode::Auto, _ => unreachable!("endpoint modes were validated above"), }; @@ -474,7 +448,6 @@ pub fn endpoint_path_matches(pattern: &str, path: &str) -> bool { pub fn parse_tls_mode(val: ®orus::Value) -> TlsMode { match get_object_str(val, "tls").as_deref() { Some("skip") => TlsMode::Skip, - // "terminate" and "passthrough" are deprecated aliases (logged by parse_l7_config); fall through to Auto. _ => TlsMode::Auto, } } @@ -1277,6 +1250,12 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< ); let loc = format!("{name}.endpoints[{i}]"); + errors.extend( + validate_endpoint_modes(tls, enforcement, access) + .into_iter() + .map(|reason| format!("{loc}: {reason}")), + ); + if protocol == "mcp" { if host.trim().is_empty() { errors.push(format!( @@ -1493,13 +1472,6 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< } } - // Deprecated tls values: warn but don't error - if tls == "terminate" || tls == "passthrough" { - warnings.push(format!( - "{loc}: 'tls: {tls}' is deprecated; TLS termination is now automatic. Use 'tls: skip' to disable." - )); - } - // tls: skip with L7 on port 443 won't work if tls == "skip" && !protocol.is_empty() && ports.contains(&443) { warnings.push(format!( @@ -1514,10 +1486,6 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< )); } - // port 443 + rest + tls: skip — L7 won't work (already handled above) - // The old warning about missing `tls: terminate` is no longer needed - // because TLS termination is now automatic. - // Per-rule deny_rules validation (semantic checks handled by // shared validator above). if has_deny_rules { @@ -1957,12 +1925,11 @@ mod tests { #[test] fn parse_l7_config_rest_enforce() { let val = regorus::Value::from_json_str( - r#"{"protocol": "rest", "tls": "terminate", "enforcement": "enforce", "host": "api.example.com", "port": 443}"#, + r#"{"protocol": "rest", "enforcement": "enforce", "host": "api.example.com", "port": 443}"#, ) .unwrap(); let config = parse_l7_config(&val).unwrap(); assert_eq!(config.protocol, L7Protocol::Rest); - // "terminate" is deprecated and treated as Auto. assert_eq!(config.tls, TlsMode::Auto); assert_eq!(config.enforcement, EnforcementMode::Enforce); } @@ -3425,30 +3392,37 @@ mod tests { } #[test] - fn validate_tls_terminate_deprecated_warning() { - let data = serde_json::json!({ - "network_policies": { - "test": { - "endpoints": [{ - "host": "api.example.com", - "port": 443, - "tls": "terminate", - "protocol": "rest", - "access": "full" - }], - "binaries": [] + fn validate_rejects_unknown_endpoint_modes_without_warning() { + for (field, value) in [ + ("tls", "terminate"), + ("tls", "passthrough"), + ("enforcement", "enforcee"), + ("access", "read_only"), + ] { + let mut endpoint = serde_json::json!({ + "host": "api.example.com", + "port": 443, + "protocol": "rest", + "access": "full" + }); + endpoint[field] = value.into(); + let data = serde_json::json!({ + "network_policies": { + "test": { "endpoints": [endpoint], "binaries": [] } } - } - }); - let (errors, warnings) = validate_l7_policies(&data); - assert!( - errors.is_empty(), - "deprecated tls should not error: {errors:?}" - ); - assert!( - warnings.iter().any(|w| w.contains("deprecated")), - "should warn about deprecated tls: {warnings:?}" - ); + }); + + let (errors, warnings) = validate_l7_policies(&data); + assert!( + errors.iter().any(|e| e.contains("test.endpoints[0]") + && e.contains(&format!("unknown {field} value '{value}'"))), + "{field}: {value} should be rejected: {errors:?}" + ); + assert!( + !warnings.iter().any(|w| w.contains("deprecated")), + "{field}: {value} should not warn: {warnings:?}" + ); + } } #[test] diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index f532b6ee64..77a86095ef 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -8575,12 +8575,12 @@ mod tests { /// placeholders in request headers before forwarding to upstream. /// /// This is the code path exercised when an endpoint has `protocol: rest` - /// and `tls: terminate` — the proxy terminates TLS, sees plaintext HTTP, - /// and replaces placeholder tokens with real secrets. + /// and terminated TLS — the proxy sees plaintext HTTP and replaces + /// placeholder tokens with real secrets. /// - /// Without this test, a misconfigured endpoint (missing `tls: terminate`) - /// silently leaks placeholder strings like `openshell:resolve:env:NVIDIA_API_KEY` - /// to the upstream API, causing 401 Unauthorized errors. + /// Without this test, a misconfigured endpoint silently leaks placeholder + /// strings like `openshell:resolve:env:NVIDIA_API_KEY` to the upstream + /// API, causing 401 Unauthorized errors. #[tokio::test] async fn relay_request_with_resolver_rewrites_credential_placeholders() { let provider_env: HashMap = [( diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 14766c7df8..9db54fa900 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -9626,7 +9626,6 @@ network_policies: port: 8080 protocol: rest enforcement: enforce - tls: terminate rules: - allow: method: GET @@ -9669,7 +9668,6 @@ network_policies: port: 8080 protocol: rest enforcement: enforce - tls: terminate rules: - allow: method: GET @@ -9717,7 +9715,6 @@ network_policies: ports: [8080, 9090] protocol: rest enforcement: enforce - tls: terminate rules: - allow: method: GET diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index 5afa4597ff..6a6c0f9d26 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -1131,6 +1131,14 @@ fn network_endpoint_from_json( { return Err(reason.to_string()); } + let mode_errors = openshell_policy::validate_endpoint_modes( + &endpoint.tls, + &endpoint.enforcement, + &endpoint.access, + ); + if !mode_errors.is_empty() { + return Err(mode_errors.join("; ")); + } let mut ports = endpoint.ports; if ports.is_empty() && endpoint.port > 0 { @@ -1429,7 +1437,6 @@ mod tests { "host": "api.github.com", "port": 443, "protocol": "rest", - "tls": "terminate", "enforcement": "enforce", "rules": [ { @@ -1527,6 +1534,28 @@ mod tests { } } + #[test] + fn proposal_chunks_from_body_rejects_removed_tls_values() { + for tls in ["terminate", "passthrough"] { + let body = format!( + r#"{{ + "operations": [{{ + "addRule": {{ + "ruleName": "bad_mode", + "rule": {{"endpoints": [{{"host":"api.example.com","port":443,"tls":"{tls}"}}]}} + }} + }}] + }}"# + ); + + let error = proposal_chunks_from_body(body.as_bytes()).unwrap_err(); + assert!( + error.contains(&format!("unknown tls value '{tls}'")), + "tls: {tls} unexpected error: {error}" + ); + } + } + #[test] fn proposal_chunks_from_body_accepts_omitted_protocol_with_default_tls() { let body = br#"{ diff --git a/crates/openshell-tui/src/lib.rs b/crates/openshell-tui/src/lib.rs index 03209e969a..0ede05d87b 100644 --- a/crates/openshell-tui/src/lib.rs +++ b/crates/openshell-tui/src/lib.rs @@ -1272,10 +1272,6 @@ fn render_policy_lines( // Rule header — include L7/TLS/allowed_ips annotation if any endpoint has it. let has_l7 = rule.endpoints.iter().any(|e| !e.protocol.is_empty()); - let has_tls_term = rule - .endpoints - .iter() - .any(|e| openshell_policy::network_tls_mode_to_str(e.tls) == Some("terminate")); let has_allowed_ips = rule.endpoints.iter().any(|e| !e.allowed_ips.is_empty()); let mut annotations = Vec::new(); if has_l7 { @@ -1289,9 +1285,6 @@ fn render_policy_lines( annotations.push(format!("L7 {proto}")); } } - if has_tls_term { - annotations.push("TLS terminate".to_string()); - } if has_allowed_ips { annotations.push("private IP".to_string()); } diff --git a/docs/about/release-notes.mdx b/docs/about/release-notes.mdx index 3b9d3f5613..62a3094b87 100644 --- a/docs/about/release-notes.mdx +++ b/docs/about/release-notes.mdx @@ -29,6 +29,18 @@ openshell sandbox create --from registry.example.com/agents/my-agent:1.0 -- my-a Gateway operators can replace the default with the driver configuration or the Helm `server.sandboxImage` value. +### Network endpoint TLS handling + +OpenShell 0.1.0 closes the network endpoint `tls` field to two forms: omit it to auto-detect and terminate TLS for inspection, or set `skip` to stop inspecting that endpoint. The gateway, the sandbox supervisor, provider profile lint, and agent-authored policy proposals all reject any other value, including the former `terminate` and `passthrough` spellings. + +Remove the `tls` field from affected endpoints before upgrading. Removal preserves behavior, because `terminate` and `passthrough` already selected automatic termination. + +Do not substitute `tls: skip` for a removed `terminate`. `skip` stops TLS inspection, credential placeholder rewriting, dynamic token grant injection, and L7 rule enforcement for that endpoint, so it widens what the endpoint allows. + +A stored policy carrying a removed value fails validation on read, and affected sandboxes cannot start. Run `openshell policy list --global` to see which revision is invalid and why, then re-apply a corrected policy with `openshell policy set --global`. + +A policy baked into a workload image at `/etc/openshell/policy.yaml` is rejected the same way, because the supervisor requests it over the sandbox boundary when the gateway has no policy for the sandbox. Setting a policy with `openshell policy set --global` repairs that case too; rebuilding the image without the field restores the original behavior. + ### Provider profiles are import-only OpenShell 0.1.0 stops compiling provider profiles into the gateway binary. A gateway's catalog contains exactly the profiles an operator imported, the default `provider_profile_sources` is `[{ type = "user" }]`, and the `builtin` source type is removed: a `gateway.toml` that still names it is rejected at startup with the import command. A gateway with nothing imported reaches ready and serves an empty catalog. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 5e03c8a927..3e1539b7b0 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -212,7 +212,7 @@ Each endpoint defines a reachable destination and optional inspection rules. | `port` | integer | Yes | TCP port number. | | `path` | string | No | Optional HTTP path glob used to select between L7 endpoints that share the same host and port. Empty means all paths. Use this when REST and GraphQL live under the same host, such as `/repos/**` and `/graphql`. | | `protocol` | string | No | Set to `tcp` with a valid DNS hostname to allow native TCP clients through policy DNS and transparent capture without payload inspection. Omit the field for L4 passthrough through an explicit proxy, including legacy hostless `allowed_ips` endpoints. Set to `rest` for HTTP method/path inspection, `websocket` for RFC 6455 upgrade and client text-message inspection, `graphql` for GraphQL-over-HTTP operation inspection, `mcp` for MCP Streamable HTTP request inspection, or `json-rpc` for generic JSON-RPC-over-HTTP method inspection. WebSocket endpoints can also use GraphQL operation rules for GraphQL-over-WebSocket traffic. Provider-credentialed endpoints require an inspected protocol unless `allow_uninspected_credentials` is explicitly set. | -| `tls` | string | No | TLS handling mode. The proxy auto-detects TLS by peeking the first bytes of each connection and terminates it for inspected HTTPS traffic, so this field is optional in most cases. Set to `skip` to disable auto-detection for edge cases such as client-certificate mTLS or non-standard protocols. Provider-credentialed endpoints reject `tls: skip` unless `allow_uninspected_credentials` is explicitly set. The values `terminate` and `passthrough` are deprecated and log a warning; they are still accepted for backward compatibility but have no effect on behavior. Remove either value and omit `tls` to use the equivalent automatic handling. Other values are rejected before activation. | +| `tls` | string | No | TLS handling mode. The proxy auto-detects TLS by peeking the first bytes of each connection and terminates it for inspected HTTPS traffic, so this field is optional in most cases. Set to `skip` to disable auto-detection for edge cases such as client-certificate mTLS or non-standard protocols. Provider-credentialed endpoints reject `tls: skip` unless `allow_uninspected_credentials` is explicitly set. `skip` is the only accepted non-empty value; every other value, including the removed `terminate` and `passthrough` spellings, is rejected before persistence or activation. | | `enforcement` | string | No | `enforce` actively blocks disallowed requests. `audit` logs violations but allows traffic through. Other values are rejected before activation rather than interpreted as audit mode. | | `access` | string | No | Access preset. One of `read-only`, `read-write`, or `full`; other values are rejected before activation. Mutually exclusive with `rules`. Not valid on `protocol: mcp` or `protocol: json-rpc`; MCP uses explicit rules unless `mcp.allow_all_known_mcp_methods: true` enables the endpoint method profile, and JSON-RPC always uses explicit rules. | | `rules` | list of allow rule objects | No | Fine-grained protocol-specific allow rules. Mutually exclusive with `access`. | diff --git a/examples/sandbox-policy-quickstart/README.md b/examples/sandbox-policy-quickstart/README.md index ce6b16bfb3..dd19bc14e2 100644 --- a/examples/sandbox-policy-quickstart/README.md +++ b/examples/sandbox-policy-quickstart/README.md @@ -98,7 +98,6 @@ network_policies: - host: api.github.com port: 443 protocol: rest - tls: terminate enforcement: enforce access: read-only binaries: @@ -111,7 +110,7 @@ select it. These settings are required because `policy set` replaces the entire policy. The `network_policies` section is the interesting part: **curl may make GET, HEAD, and OPTIONS requests to `api.github.com` over HTTPS. -Everything else is denied.** The proxy terminates TLS (`tls: terminate`) +Everything else is denied.** The proxy auto-detects and terminates TLS to inspect each HTTP request and enforce the `read-only` access preset at the method level. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 4edb490543..1d837b7f0e 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -108,9 +108,9 @@ enum NetworkTlsMode { NETWORK_TLS_MODE_UNSPECIFIED = 0; // Disable TLS detection and relay the connection without inspection. NETWORK_TLS_MODE_SKIP = 1; - // Deprecated compatibility spelling; use NETWORK_TLS_MODE_UNSPECIFIED. + // Rejected by policy validation; use NETWORK_TLS_MODE_UNSPECIFIED. NETWORK_TLS_MODE_TERMINATE = 2 [deprecated = true]; - // Deprecated compatibility spelling; use NETWORK_TLS_MODE_UNSPECIFIED. + // Rejected by policy validation; use NETWORK_TLS_MODE_UNSPECIFIED. NETWORK_TLS_MODE_PASSTHROUGH = 3 [deprecated = true]; } diff --git a/providers/aws-s3.yaml b/providers/aws-s3.yaml index 0cd973e6b7..89a60a3f30 100644 --- a/providers/aws-s3.yaml +++ b/providers/aws-s3.yaml @@ -93,7 +93,6 @@ endpoints: - host: "*.s3.amazonaws.com" port: 443 protocol: rest - tls: terminate access: read-write enforcement: enforce credential_signing: sigv4 @@ -101,7 +100,6 @@ endpoints: - host: "s3.amazonaws.com" port: 443 protocol: rest - tls: terminate access: read-write enforcement: enforce credential_signing: sigv4 @@ -109,7 +107,6 @@ endpoints: - host: "*.s3.*.amazonaws.com" port: 443 protocol: rest - tls: terminate access: read-write enforcement: enforce credential_signing: sigv4 @@ -117,7 +114,6 @@ endpoints: - host: "s3.*.amazonaws.com" port: 443 protocol: rest - tls: terminate access: read-write enforcement: enforce credential_signing: sigv4 @@ -125,7 +121,6 @@ endpoints: - host: "*.s3.dualstack.*.amazonaws.com" port: 443 protocol: rest - tls: terminate access: read-write enforcement: enforce credential_signing: sigv4 @@ -133,7 +128,6 @@ endpoints: - host: "s3.dualstack.*.amazonaws.com" port: 443 protocol: rest - tls: terminate access: read-write enforcement: enforce credential_signing: sigv4 diff --git a/sdk/go/openshell/v1/types/network_policy.go b/sdk/go/openshell/v1/types/network_policy.go index bd8baf8d14..0e413483f7 100644 --- a/sdk/go/openshell/v1/types/network_policy.go +++ b/sdk/go/openshell/v1/types/network_policy.go @@ -21,9 +21,9 @@ const ( NetworkTLSModeUnspecified NetworkTLSMode = 0 // NetworkTLSModeSkip disables TLS inspection. NetworkTLSModeSkip NetworkTLSMode = 1 - // NetworkTLSModeTerminate is retained for wire compatibility; prefer unspecified. + // NetworkTLSModeTerminate is rejected by policy validation. NetworkTLSModeTerminate NetworkTLSMode = 2 - // NetworkTLSModePassthrough is retained for wire compatibility; prefer unspecified. + // NetworkTLSModePassthrough is rejected by policy validation. NetworkTLSModePassthrough NetworkTLSMode = 3 ) diff --git a/sdk/go/openshell/v1/types_reexport.go b/sdk/go/openshell/v1/types_reexport.go index ae1878a6c9..090918b6a0 100644 --- a/sdk/go/openshell/v1/types_reexport.go +++ b/sdk/go/openshell/v1/types_reexport.go @@ -29,9 +29,9 @@ const ( NetworkTLSModeUnspecified = types.NetworkTLSModeUnspecified // NetworkTLSModeSkip disables TLS inspection. NetworkTLSModeSkip = types.NetworkTLSModeSkip - // NetworkTLSModeTerminate is retained for wire compatibility. + // NetworkTLSModeTerminate is rejected by policy validation. NetworkTLSModeTerminate = types.NetworkTLSModeTerminate - // NetworkTLSModePassthrough is retained for wire compatibility. + // NetworkTLSModePassthrough is rejected by policy validation. NetworkTLSModePassthrough = types.NetworkTLSModePassthrough // NetworkEnforcementModeUnspecified uses the documented audit default. diff --git a/sdk/go/proto/sandboxv1/sandbox.pb.go b/sdk/go/proto/sandboxv1/sandbox.pb.go index 08434bdc1c..c38db28c2f 100644 --- a/sdk/go/proto/sandboxv1/sandbox.pb.go +++ b/sdk/go/proto/sandboxv1/sandbox.pb.go @@ -35,11 +35,11 @@ const ( NetworkTlsMode_NETWORK_TLS_MODE_UNSPECIFIED NetworkTlsMode = 0 // Disable TLS detection and relay the connection without inspection. NetworkTlsMode_NETWORK_TLS_MODE_SKIP NetworkTlsMode = 1 - // Deprecated compatibility spelling; use NETWORK_TLS_MODE_UNSPECIFIED. + // Rejected by policy validation; use NETWORK_TLS_MODE_UNSPECIFIED. // // Deprecated: Marked as deprecated in sandbox.proto. NetworkTlsMode_NETWORK_TLS_MODE_TERMINATE NetworkTlsMode = 2 - // Deprecated compatibility spelling; use NETWORK_TLS_MODE_UNSPECIFIED. + // Rejected by policy validation; use NETWORK_TLS_MODE_UNSPECIFIED. // // Deprecated: Marked as deprecated in sandbox.proto. NetworkTlsMode_NETWORK_TLS_MODE_PASSTHROUGH NetworkTlsMode = 3 diff --git a/skills/generate-sandbox-policy/SKILL.md b/skills/generate-sandbox-policy/SKILL.md index 37e1438cb7..98a21f162e 100644 --- a/skills/generate-sandbox-policy/SKILL.md +++ b/skills/generate-sandbox-policy/SKILL.md @@ -205,13 +205,9 @@ Is L7 inspection needed? ### TLS Decision -| API host port | TLS setting | -|--------------|-------------| -| Port 443 (HTTPS) and L7 rules/preset needed | `tls: terminate` (required for inspection) | -| Port 443 (HTTPS) and L4-only | Omit `tls` (passthrough, no L7); choose omitted protocol or explicit TCP based on client/runtime as above | -| Non-443 (HTTP) | Omit `tls` | +Omit `tls` on every endpoint, regardless of port: the proxy auto-detects TLS and terminates it for inspection. `skip` is the only accepted non-empty value, reserved for upstreams requiring client-certificate mTLS or a non-HTTP protocol. -**Critical**: `protocol: rest` on port 443 without `tls: terminate` will not work — the proxy cannot inspect encrypted traffic. Always set `tls: terminate` when combining port 443 with L7 rules. +Do not "fix" a rejected value — including the removed `terminate` and `passthrough` spellings — by changing it to `skip`; remove the field instead. `skip` stops inspection, credential injection, and L7 rule enforcement for that endpoint, so it silently widens what the endpoint allows. ### Middleware Decision @@ -274,7 +270,6 @@ network_policies: - host: port: protocol: rest # Required for L7 inspection - tls: terminate # Required for HTTPS + L7 enforcement: enforce # or audit # Use ONE of: access OR rules (never both) access: # read-only | read-write | full @@ -378,7 +373,7 @@ Before presenting the policy to the user, verify correctness **and** flag breadt `protocol: tcp` is L4-only and must not contain either field - [ ] Every `protocol: tcp` endpoint has a valid DNS hostname; it is not hostless, an IP literal, a trailing-dot name, or a malformed DNS selector -- [ ] If `tls: terminate` is set, `protocol` is also set +- [ ] `tls` is either omitted or set to `skip`; no other value is accepted - [ ] `rules` list is not empty when present - [ ] If `protocol: sql`, `enforcement` is not `enforce` - [ ] Every middleware config has a non-empty `middleware` name and non-empty `endpoints.include` @@ -390,7 +385,7 @@ Before presenting the policy to the user, verify correctness **and** flag breadt ### Schema Warnings (log-only, but should be fixed) -- [ ] `protocol: rest` on port 443 should have `tls: terminate` +- [ ] `tls: skip` is not combined with L7 rules on port 443; inspection cannot work on encrypted traffic - [ ] HTTP methods are standard: GET, HEAD, POST, PUT, DELETE, PATCH, OPTIONS, or `*` - [ ] Credentialed destinations are also covered by the attached provider profile endpoint; policy admission alone does not authorize credential @@ -557,7 +552,6 @@ my_api_readonly: - host: api.example.com port: 443 protocol: rest - tls: terminate enforcement: enforce access: read-only binaries: @@ -573,7 +567,6 @@ my_api_custom: - host: api.example.com port: 443 protocol: rest - tls: terminate enforcement: enforce rules: - allow: diff --git a/skills/generate-sandbox-policy/examples.md b/skills/generate-sandbox-policy/examples.md index 2cbc21b6b6..f5475a6c2a 100644 --- a/skills/generate-sandbox-policy/examples.md +++ b/skills/generate-sandbox-policy/examples.md @@ -3,11 +3,10 @@ Examples organized by detail tier — from minimal (just host + intent) to full (complete API docs). > **TLS note:** TLS termination is automatic. The proxy auto-detects TLS by -> peeking the first bytes of each connection, so there is no need to specify -> `tls: terminate` in policies. The `tls: terminate` and `tls: passthrough` -> values are deprecated. If you have an edge case where auto-detection must -> be bypassed, you can set `tls: skip` to disable TLS interception for that -> endpoint. +> peeking the first bytes of each connection, so omit `tls` on inspected +> endpoints. `skip` is the only accepted value and disables TLS interception +> for that endpoint; the removed `terminate` and `passthrough` spellings fail +> policy validation. --- From 4b1c09de28eb012a8d83476dddf92c0fc69bd03c Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 22 Sep 2026 19:04:59 +0000 Subject: [PATCH 07/10] fix(network): preserve chunked request boundaries (#3530) * fix(network): preserve chunked request boundaries Signed-off-by: Piotr Mlocek * fix(network): bound chunk framing amplification Signed-off-by: Piotr Mlocek * fix(supervisor): box sandbox runtime future Signed-off-by: Piotr Mlocek * fix(network): buffer chunked relay read-ahead Signed-off-by: Piotr Mlocek * fix(network): flush completed chunks promptly Signed-off-by: Piotr Mlocek --------- Signed-off-by: Piotr Mlocek --- .../src/l7/relay.rs | 22 ++ .../src/l7/rest.rs | 337 ++++++++++++++++-- e2e/rust/tests/proxy_egress_pipeline.rs | 62 ++++ 3 files changed, 392 insertions(+), 29 deletions(-) diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 132b7674d4..574ad3be8f 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -35,6 +35,8 @@ use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::{debug, warn}; +const CONNECTION_READ_AHEAD_BYTES: usize = 8 * 1024; + /// Context for L7 request policy evaluation. #[derive(Clone)] #[cfg_attr(test, derive(Default))] @@ -754,6 +756,17 @@ where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, { + // Keep read-ahead state for the lifetime of the inspected connection. An + // HTTP parser may fetch bytes from the next pipelined request while + // finishing the current one; retaining them here ensures the next request + // still passes through its own policy decision. + let mut client_buffer = + tokio::io::BufReader::with_capacity(CONNECTION_READ_AHEAD_BYTES, client); + let mut upstream_buffer = + tokio::io::BufReader::with_capacity(CONNECTION_READ_AHEAD_BYTES, upstream); + let client = &mut client_buffer; + let upstream = &mut upstream_buffer; + match config.protocol { L7Protocol::Rest | L7Protocol::Websocket => { relay_rest(config, &engine, client, upstream, ctx).await @@ -813,6 +826,15 @@ where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, { + // Route selection also owns the full keep-alive loop, so buffered bytes + // remain available across per-request parsing and authorization. + let mut client_buffer = + tokio::io::BufReader::with_capacity(CONNECTION_READ_AHEAD_BYTES, client); + let mut upstream_buffer = + tokio::io::BufReader::with_capacity(CONNECTION_READ_AHEAD_BYTES, upstream); + let client = &mut client_buffer; + let upstream = &mut upstream_buffer; + let provider = crate::l7::rest::RestProvider::with_options(crate::l7::path::CanonicalizeOptions { allow_encoded_slash: configs.iter().any(|config| config.allow_encoded_slash), diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 77a86095ef..f8b1f3166f 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -3128,6 +3128,7 @@ where let started_at = std::time::Instant::now(); let mut read_buf = [0u8; RELAY_BUF_SIZE]; let mut parse_buf = Vec::from(already_forwarded); + let mut forwarded_len = already_forwarded.len(); let mut pos = 0usize; let mut chunk_count = 0usize; let mut chunk_payload_bytes = 0usize; @@ -3140,16 +3141,26 @@ where if let Some(end) = find_crlf(&parse_buf, pos) { break end; } - let n = reader.read(&mut read_buf).await.into_diagnostic()?; - if n == 0 { - return Err(miette!("Chunked body ended before chunk-size line")); + if parse_buf.len().saturating_sub(pos) >= MAX_HEADER_BYTES { + return Err(miette!("Chunk-size line exceeds limit")); } - if let Some(guard) = generation_guard { - guard.ensure_current()?; - } - writer.write_all(&read_buf[..n]).await.into_diagnostic()?; - parse_buf.extend_from_slice(&read_buf[..n]); + let target_len = parse_buf + .len() + .checked_add(1) + .ok_or_else(|| miette!("Chunked body size overflow"))?; + relay_chunked_until_len( + reader, + &mut read_buf, + &mut parse_buf, + target_len, + generation_guard, + "Chunked body ended before chunk-size line", + ) + .await?; }; + if size_line_end.saturating_add(2).saturating_sub(pos) > MAX_HEADER_BYTES { + return Err(miette!("Chunk-size line exceeds limit")); + } let size_line = std::str::from_utf8(&parse_buf[pos..size_line_end]) .into_diagnostic() @@ -3163,6 +3174,15 @@ where .into_diagnostic() .map_err(|_| miette!("Invalid chunk size token: {size_token:?}"))?; pos = size_line_end + 2; + flush_validated_chunked_bytes( + writer, + &parse_buf, + &mut forwarded_len, + pos, + false, + generation_guard, + ) + .await?; if chunk_size == 0 { // Parse trailers (if any). Terminates on empty trailer line. @@ -3172,20 +3192,40 @@ where if let Some(end) = find_crlf(&parse_buf, pos) { break end; } - let n = reader.read(&mut read_buf).await.into_diagnostic()?; - if n == 0 { - return Err(miette!("Chunked body ended before trailer terminator")); - } - if let Some(guard) = generation_guard { - guard.ensure_current()?; + if parse_buf.len().saturating_sub(pos) >= MAX_HEADER_BYTES { + return Err(miette!("Chunk trailer line exceeds limit")); } - writer.write_all(&read_buf[..n]).await.into_diagnostic()?; - parse_buf.extend_from_slice(&read_buf[..n]); + let target_len = parse_buf + .len() + .checked_add(1) + .ok_or_else(|| miette!("Chunked trailer size overflow"))?; + relay_chunked_until_len( + reader, + &mut read_buf, + &mut parse_buf, + target_len, + generation_guard, + "Chunked body ended before trailer terminator", + ) + .await?; }; + if trailer_end.saturating_add(2).saturating_sub(pos) > MAX_HEADER_BYTES { + return Err(miette!("Chunk trailer line exceeds limit")); + } let trailer_line = &parse_buf[pos..trailer_end]; + let trailer_is_empty = trailer_line.is_empty(); pos = trailer_end + 2; - if trailer_line.is_empty() { + flush_validated_chunked_bytes( + writer, + &parse_buf, + &mut forwarded_len, + pos, + trailer_is_empty, + generation_guard, + ) + .await?; + if trailer_is_empty { debug!( chunk_count, chunk_payload_bytes, @@ -3196,6 +3236,11 @@ where return Ok(()); } trailer_count += 1; + if pos > RELAY_BUF_SIZE * 4 && forwarded_len >= pos { + parse_buf.drain(..pos); + forwarded_len -= pos; + pos = 0; + } } } @@ -3207,32 +3252,106 @@ where .checked_add(2) .ok_or_else(|| miette!("Chunk size overflow"))?; - while parse_buf.len() < chunk_with_crlf_end { - let n = reader.read(&mut read_buf).await.into_diagnostic()?; - if n == 0 { - return Err(miette!("Chunked body ended mid-chunk")); - } - if let Some(guard) = generation_guard { - guard.ensure_current()?; - } - writer.write_all(&read_buf[..n]).await.into_diagnostic()?; - parse_buf.extend_from_slice(&read_buf[..n]); - } + relay_chunked_until_len( + reader, + &mut read_buf, + &mut parse_buf, + chunk_with_crlf_end, + generation_guard, + "Chunked body ended mid-chunk", + ) + .await?; if &parse_buf[chunk_end..chunk_with_crlf_end] != b"\r\n" { return Err(miette!("Chunk missing terminating CRLF")); } pos = chunk_with_crlf_end; + flush_validated_chunked_bytes( + writer, + &parse_buf, + &mut forwarded_len, + pos, + true, + generation_guard, + ) + .await?; chunk_count += 1; chunk_payload_bytes = chunk_payload_bytes.saturating_add(chunk_size); // Keep parser memory bounded for long streams. - if pos > RELAY_BUF_SIZE * 4 { + if pos > RELAY_BUF_SIZE * 4 && forwarded_len >= pos { parse_buf.drain(..pos); + forwarded_len -= pos; pos = 0; } } } +/// Read only the bytes needed to reach `target_len`. +/// +/// The connection-scoped buffered reader may fetch more data from the socket, +/// but this parser consumes only the current body. Any read-ahead remains in +/// that reader for the next request's policy decision. +async fn relay_chunked_until_len( + reader: &mut R, + read_buf: &mut [u8; RELAY_BUF_SIZE], + parse_buf: &mut Vec, + target_len: usize, + generation_guard: Option<&PolicyGenerationGuard>, + eof_message: &'static str, +) -> Result<()> +where + R: AsyncRead + Unpin, +{ + while parse_buf.len() < target_len { + let remaining = target_len - parse_buf.len(); + let to_read = remaining.min(read_buf.len()); + let n = reader + .read(&mut read_buf[..to_read]) + .await + .into_diagnostic()?; + if n == 0 { + return Err(miette!(eof_message)); + } + if let Some(guard) = generation_guard { + guard.ensure_current()?; + } + parse_buf.extend_from_slice(&read_buf[..n]); + } + Ok(()) +} + +/// Forward complete, validated chunk framing. +/// +/// Size lines may be coalesced with their payload, but every completed chunk is +/// forwarded immediately so streaming request and response bodies make +/// progress without waiting for the terminal chunk. This writes by validated +/// framing units rather than once per framing byte. +async fn flush_validated_chunked_bytes( + writer: &mut W, + parse_buf: &[u8], + forwarded_len: &mut usize, + validated_len: usize, + force: bool, + generation_guard: Option<&PolicyGenerationGuard>, +) -> Result<()> +where + W: AsyncWrite + Unpin, +{ + let pending = validated_len.saturating_sub(*forwarded_len); + if pending == 0 || (!force && pending < RELAY_BUF_SIZE) { + return Ok(()); + } + if let Some(guard) = generation_guard { + guard.ensure_current()?; + } + writer + .write_all(&parse_buf[*forwarded_len..validated_len]) + .await + .into_diagnostic()?; + *forwarded_len = validated_len; + Ok(()) +} + fn find_crlf(buf: &[u8], start: usize) -> Option { buf.get(start..)? .windows(2) @@ -3857,6 +3976,38 @@ mod tests { } } + #[derive(Default)] + struct CountingWriter { + writes: usize, + bytes: Vec, + } + + impl AsyncWrite for CountingWriter { + fn poll_write( + mut self: Pin<&mut Self>, + _context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + self.writes += 1; + self.bytes.extend_from_slice(buffer); + Poll::Ready(Ok(buffer.len())) + } + + fn poll_flush( + self: Pin<&mut Self>, + _context: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + _context: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + } + fn write_header(name: &str, value: &str, on_existing: ExistingHeaderAction) -> HeaderMutation { HeaderMutation { operation: Some(header_mutation::Operation::Write( @@ -4559,6 +4710,134 @@ mod tests { } } + #[tokio::test] + async fn relay_chunked_leaves_pipelined_request_for_next_policy_decision() { + let chunked_body = b"0\r\n\r\n"; + let pipelined_request = + b"DELETE /blocked HTTP/1.1\r\nHost: example.com\r\nContent-Length: 0\r\n\r\n"; + let mut wire = chunked_body.to_vec(); + wire.extend_from_slice(pipelined_request); + + let (relay_reader, mut client_writer) = tokio::io::duplex(4096); + client_writer.write_all(&wire).await.unwrap(); + let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); + let (mut relay_writer, mut upstream_reader) = tokio::io::duplex(4096); + + relay_chunked(&mut relay_reader, &mut relay_writer, &[], None) + .await + .expect("chunked body should relay"); + + let mut remaining = vec![0; pipelined_request.len()]; + relay_reader.read_exact(&mut remaining).await.unwrap(); + assert_eq!(remaining, pipelined_request); + + drop(relay_writer); + let mut forwarded = Vec::new(); + upstream_reader.read_to_end(&mut forwarded).await.unwrap(); + assert_eq!(forwarded, chunked_body); + } + + #[tokio::test] + async fn relay_chunked_with_forwarded_prefix_and_trailers_preserves_pipeline_boundary() { + let already_forwarded = b"3\r\na"; + let body_remainder = b"bc\r\n0\r\nX-Checksum: abc123\r\n\r\n"; + let pipelined_request = + b"DELETE /blocked HTTP/1.1\r\nHost: example.com\r\nContent-Length: 0\r\n\r\n"; + let mut later_read = body_remainder.to_vec(); + later_read.extend_from_slice(pipelined_request); + + let (relay_reader, mut client_writer) = tokio::io::duplex(4096); + client_writer.write_all(&later_read).await.unwrap(); + let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); + let (mut relay_writer, mut upstream_reader) = tokio::io::duplex(4096); + relay_writer.write_all(already_forwarded).await.unwrap(); + + relay_chunked( + &mut relay_reader, + &mut relay_writer, + already_forwarded, + None, + ) + .await + .expect("chunked body with trailers should relay"); + + let mut remaining = vec![0; pipelined_request.len()]; + relay_reader.read_exact(&mut remaining).await.unwrap(); + assert_eq!(remaining, pipelined_request); + + drop(relay_writer); + let mut forwarded = Vec::new(); + upstream_reader.read_to_end(&mut forwarded).await.unwrap(); + let mut expected = already_forwarded.to_vec(); + expected.extend_from_slice(body_remainder); + assert_eq!(forwarded, expected); + } + + #[tokio::test] + async fn relay_chunked_handles_tiny_chunks_without_an_aggregate_framing_limit() { + let mut chunked_body = Vec::new(); + for _ in 0..10_000 { + chunked_body.extend_from_slice(b"1\r\na\r\n"); + } + chunked_body.extend_from_slice(b"0\r\n\r\n"); + let pipelined_request = b"DELETE /blocked HTTP/1.1\r\nHost: example.com\r\n\r\n"; + let mut wire = chunked_body.clone(); + wire.extend_from_slice(pipelined_request); + + let mut reader = + tokio::io::BufReader::with_capacity(RELAY_BUF_SIZE, CountingReader::new(wire)); + let mut writer = CountingWriter::default(); + relay_chunked(&mut reader, &mut writer, &[], None) + .await + .expect("valid tiny chunks must not be rejected by an aggregate framing limit"); + + let reads_after_body = reader.get_ref().reads; + let mut remaining = vec![0; pipelined_request.len()]; + reader.read_exact(&mut remaining).await.unwrap(); + assert_eq!(remaining, pipelined_request); + assert_eq!(writer.bytes, chunked_body); + assert!( + reads_after_body < 100, + "connection buffering required {reads_after_body} underlying reads" + ); + assert!( + writer.writes <= 10_001, + "chunk forwarding should require at most one write per chunk, got {}", + writer.writes + ); + } + + #[tokio::test] + async fn relay_chunked_forwards_complete_chunk_before_stream_ends() { + let (relay_reader, mut source_writer) = tokio::io::duplex(4096); + let (relay_writer, mut destination_reader) = tokio::io::duplex(4096); + + let relay = tokio::spawn(async move { + let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); + let mut relay_writer = relay_writer; + relay_chunked(&mut relay_reader, &mut relay_writer, &[], None).await + }); + + let first_chunk = b"5\r\nhello\r\n"; + source_writer.write_all(first_chunk).await.unwrap(); + + let mut forwarded = vec![0; first_chunk.len()]; + tokio::time::timeout( + std::time::Duration::from_secs(2), + destination_reader.read_exact(&mut forwarded), + ) + .await + .expect("complete chunk must be forwarded while the stream remains open") + .unwrap(); + assert_eq!(forwarded, first_chunk); + + source_writer.write_all(b"0\r\n\r\n").await.unwrap(); + relay + .await + .expect("relay task must complete") + .expect("terminal chunk must relay"); + } + #[test] fn parse_no_body() { let headers = "GET /api HTTP/1.1\r\nHost: example.com\r\n\r\n"; diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs index 9c5c83a969..9556c16429 100644 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -1499,6 +1499,68 @@ print("TRANSPARENT_PIPELINE_DENIED") assert!(!observed.contains("/blocked")); } +#[tokio::test] +#[serial(proxy_egress_pipeline)] +async fn chunked_pipeline_is_authorized_separately_before_reaching_upstream() { + let server = PipelineProbeServer::start() + .await + .expect("start pipeline probe server"); + let endpoint_options = r#" protocol: rest + enforcement: enforce + rules: + - allow: + method: POST + path: "/allowed""#; + let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options) + .expect("write pipeline policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import socket + +target = "{host}:{port}" +first = ( + f"POST /allowed HTTP/1.1\r\n" + f"Host: {{target}}\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\n\r\n" + f"0\r\n\r\n" +) +second = ( + f"DELETE /blocked HTTP/1.1\r\n" + f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" +) +with socket.create_connection(({host:?}, {port}), timeout=10) as sock: + sock.sendall((first + second).encode()) + response = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk +responses = response.count(b"HTTP/1.1 ") +first_status = response.split(b"\r\n", 1)[0] +if responses != 2 or b" 200 " not in first_status or b"HTTP/1.1 403 Forbidden" not in response: + raise RuntimeError(f"unexpected chunked pipeline response: {{response!r}}") +print("CHUNKED_PIPELINE_DENIED") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + assert!( + guard.create_output.contains("CHUNKED_PIPELINE_DENIED"), + "transparent HTTP stream did not evaluate the chunked pipeline separately:\n{}", + guard.create_output + ); + + let observed = String::from_utf8(server.observed_request()).expect("upstream HTTP request"); + assert!(observed.starts_with("POST /allowed HTTP/1.1\r\n")); + assert!(observed.ends_with("0\r\n\r\n")); + assert!(!observed.contains("/blocked")); +} + #[tokio::test] #[serial(proxy_egress_pipeline)] async fn http_credentials_are_rewritten_in_transparent_headers_and_bodies() { From e367d47394e84e199908e8d426c71781aa1394ae Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 22 Sep 2026 19:15:51 +0000 Subject: [PATCH 08/10] fix(sandbox): reclaim socket descriptors before exhaustion (#3532) * fix(sandbox): reclaim socket descriptors before exhaustion Signed-off-by: Piotr Mlocek * fix(sandbox): separate socket and descriptor limits Signed-off-by: Piotr Mlocek * fix(sandbox): account for existing broker descriptors Signed-off-by: Piotr Mlocek --------- Signed-off-by: Piotr Mlocek --- .../src/linux/socket_registry.rs | 36 +++++ .../openshell-sandbox/src/network_broker.rs | 140 +++++++++++++++++- 2 files changed, 174 insertions(+), 2 deletions(-) diff --git a/crates/openshell-isolation-interface/src/linux/socket_registry.rs b/crates/openshell-isolation-interface/src/linux/socket_registry.rs index 81a71920e3..7a245d1eac 100644 --- a/crates/openshell-isolation-interface/src/linux/socket_registry.rs +++ b/crates/openshell-isolation-interface/src/linux/socket_registry.rs @@ -205,6 +205,15 @@ impl SocketRegistry { self.entries.is_empty() } + /// Number of entries that still retain a broker-owned descriptor. + #[must_use] + pub fn retained_preconnect_count(&self) -> usize { + self.entries + .values() + .filter(|entry| entry.retained_preconnect.is_some()) + .count() + } + /// Whether another socket would exceed the configured bound. #[must_use] pub fn is_full(&self) -> bool { @@ -458,4 +467,31 @@ mod tests { assert!(registry.remove_inode(first.inode)); assert!(!registry.remove_inode(second.inode)); } + + #[test] + fn retained_count_excludes_connected_metadata() { + let mut registry = SocketRegistry::new(13, 2).unwrap(); + let retained = registry + .commit(registry.stage(tcp_socket(), metadata()).unwrap()) + .unwrap(); + let connected = registry + .commit(registry.stage(tcp_socket(), metadata()).unwrap()) + .unwrap(); + + assert_eq!(registry.retained_preconnect_count(), 2); + + let entry = registry.entries.get_mut(&connected.inode).unwrap(); + entry.set_state(SocketState::Connected { + original_peer: "127.0.0.1:443".parse().unwrap(), + }); + entry.release_preconnect(); + + assert_eq!(registry.len(), 2); + assert_eq!(registry.retained_preconnect_count(), 1); + assert!( + registry.entries[&retained.inode] + .retained_preconnect + .is_some() + ); + } } diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index d4972465c6..d093ebf199 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -28,6 +28,7 @@ use openshell_isolation_interface::linux::task_memory; use tokio::sync::{mpsc, oneshot}; const SOCKET_CAPACITY: usize = 4_096; +const SOCKET_FD_HEADROOM: usize = 64; const OPEN_QUEUE_CAPACITY: usize = 256; const ACCEPT_WORKER_CAPACITY: usize = 64; const DNS_QUEUE_CAPACITY: usize = 256; @@ -196,6 +197,7 @@ struct NotificationQueues { dns_relay: DnsRelay, active_opens: Arc, active_accepts: Arc, + retained_socket_capacity: usize, decision_timeout: Duration, } @@ -252,11 +254,12 @@ impl NetworkBroker { })?); let (pending_tx, pending_rx) = mpsc::channel(OPEN_QUEUE_CAPACITY); let (pending_dns_tx, pending_dns_rx) = mpsc::channel(DNS_QUEUE_CAPACITY); - let registry = Arc::new(Mutex::new(SocketRegistry::new(1, SOCKET_CAPACITY)?)); let active_opens = Arc::new(AtomicUsize::new(0)); let active_accepts = Arc::new(AtomicUsize::new(0)); let dns_relay = start_dns_relay(dns_address, pending_dns_tx)?; let dns_address = dns_relay.address; + let retained_socket_capacity = retained_socket_capacity()?; + let registry = Arc::new(Mutex::new(SocketRegistry::new(1, SOCKET_CAPACITY)?)); let queues = NotificationQueues { protected_control_port, accept_registrar: accept_monitor.registrar(), @@ -265,6 +268,7 @@ impl NetworkBroker { dns_relay, active_opens, active_accepts, + retained_socket_capacity, decision_timeout, }; let healthy = Arc::new(AtomicBool::new(true)); @@ -539,7 +543,12 @@ fn dispatch_notification( ); } if syscall == libc::SYS_socket { - return create_socket(®istry, &listener, notification); + return create_socket( + ®istry, + &listener, + notification, + queues.retained_socket_capacity, + ); } if syscall == libc::SYS_connect { return connect_socket(registry, listener, notification, queues); @@ -587,6 +596,7 @@ fn create_socket( registry: &Mutex, listener: &NotificationListener, notification: Notification, + retained_socket_capacity: usize, ) -> io::Result<()> { let domain = i32::try_from(notification.args[0]) .map_err(|_| io::Error::from_raw_os_error(libc::EAFNOSUPPORT))?; @@ -608,6 +618,12 @@ fn create_socket( } else { InetFamily::V6 }; + // Reclaim stale descriptors before the broker exhausts its process limit. + // Connected sockets remain in the metadata registry without consuming + // this broker-owned descriptor budget. + if let Err(error) = prepare_registry_for_socket(registry, retained_socket_capacity) { + return listener.respond_errno(notification.id, error_to_errno(&error)); + } // SAFETY: arguments were reduced to the supported native INET matrix. A // successful call returns one newly owned descriptor. let mut source = unsafe { libc::socket(domain, raw_kind, protocol) }; @@ -643,6 +659,44 @@ fn create_socket( Ok(()) } +fn retained_socket_capacity() -> io::Result { + let mut limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: limit points to writable storage for one rlimit value. + if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &raw mut limit) } < 0 { + return Err(io::Error::last_os_error()); + } + let soft_limit = usize::try_from(limit.rlim_cur).unwrap_or(usize::MAX); + let open_descriptors = std::fs::read_dir("/proc/self/fd")?.count(); + Ok(retained_socket_capacity_for_limit( + soft_limit, + open_descriptors, + )) +} + +fn retained_socket_capacity_for_limit(soft_limit: usize, open_descriptors: usize) -> usize { + soft_limit + .saturating_sub(open_descriptors) + .saturating_sub(SOCKET_FD_HEADROOM) + .clamp(1, SOCKET_CAPACITY) +} + +fn prepare_registry_for_socket( + registry: &Mutex, + retained_socket_capacity: usize, +) -> io::Result<()> { + let mut registry = lock(registry); + if registry.is_full() || registry.retained_preconnect_count() >= retained_socket_capacity { + collect_closed_socket_entries_locked(&mut registry)?; + } + if registry.is_full() || registry.retained_preconnect_count() >= retained_socket_capacity { + return Err(io::Error::from_raw_os_error(libc::EMFILE)); + } + Ok(()) +} + fn reject_protected_control_destination( destination: SocketAddr, protected_port: Option, @@ -1763,6 +1817,88 @@ mod tests { use std::io::{Read as _, Write as _}; use std::os::unix::net::{UnixListener, UnixStream}; + #[test] + fn retained_socket_capacity_reserves_process_descriptor_headroom() { + assert_eq!(retained_socket_capacity_for_limit(1_024, 24), 936); + assert_eq!(retained_socket_capacity_for_limit(128, 32), 32); + assert_eq!(retained_socket_capacity_for_limit(64, 0), 1); + assert_eq!( + retained_socket_capacity_for_limit(usize::MAX, 0), + SOCKET_CAPACITY + ); + } + + #[test] + fn retained_socket_limit_reclaims_stale_entry_before_opening_another_socket() { + // SAFETY: socket returns one newly owned descriptor on success. + let fd = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + assert!(fd >= 0, "socket: {}", io::Error::last_os_error()); + // SAFETY: successful socket returned one owned descriptor. + let socket = unsafe { OwnedFd::from_raw_fd(fd) }; + let metadata = SocketMetadata { + family: InetFamily::V4, + kind: InetKind::Tcp, + close_on_exec: true, + nonblocking: false, + creator_generation: 1, + }; + let mut registry = SocketRegistry::new(1, 2).unwrap(); + let tentative = registry.stage(socket, metadata).unwrap(); + registry.commit(tentative).unwrap(); + assert!(!registry.is_full()); + assert_eq!(registry.retained_preconnect_count(), 1); + + let registry = Mutex::new(registry); + prepare_registry_for_socket(®istry, 1).unwrap(); + + assert!(lock(®istry).is_empty()); + } + + #[test] + fn retained_socket_limit_does_not_cap_connected_metadata() { + // SAFETY: socket returns one newly owned descriptor on success. + let fd = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + assert!(fd >= 0, "socket: {}", io::Error::last_os_error()); + // SAFETY: successful socket returned one owned descriptor. + let socket = unsafe { OwnedFd::from_raw_fd(fd) }; + let metadata = SocketMetadata { + family: InetFamily::V4, + kind: InetKind::Tcp, + close_on_exec: true, + nonblocking: false, + creator_generation: 1, + }; + let mut registry = SocketRegistry::new(1, 2).unwrap(); + let tentative = registry.stage(socket, metadata).unwrap(); + registry + .commit_with_state( + tentative, + SocketState::Connected { + original_peer: "127.0.0.1:443".parse().unwrap(), + }, + ) + .unwrap(); + assert_eq!(registry.len(), 1); + assert_eq!(registry.retained_preconnect_count(), 0); + + let registry = Mutex::new(registry); + prepare_registry_for_socket(®istry, 1).unwrap(); + + assert_eq!(lock(®istry).len(), 1); + } + #[test] fn notification_receive_retries_interrupted_and_disappeared_targets() { assert!(retry_notification_receive(&io::Error::from( From 88357775ad220ab2203b21caea4ec2ed86e292f0 Mon Sep 17 00:00:00 2001 From: Jim Meyer Date: Tue, 22 Sep 2026 19:29:24 +0000 Subject: [PATCH 09/10] docs(windows): align Z3 pin with z3-sys 0.13 (#3561) z3-sys 0.13 generates bindings for the Z3 5.x series and resolves GH_RELEASE_VERSION to 5.1.0 for the prebuilt-release path. The Windows wrapper still pinned Z3_SYS_Z3_VERSION to 4.16.0, so a prebuilt Windows build would fetch an archive that does not match the generated bindings. Move the wrapper pin to 5.1.0 and update the contributor, architecture, and skill references that still named 4.16.0. Signed-off-by: Jim Meyer --- .agents/skills/build-openshell-mxc-windows/SKILL.md | 2 +- .agents/skills/build-openshell-mxc-windows/reference.md | 2 +- CONTRIBUTING.md | 6 +++--- architecture/windows-msvc-build.md | 2 +- tasks/scripts/windows-msvc.ps1 | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 9b6229d96b..2bf8858a9b 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -135,7 +135,7 @@ from this skill. | `CARGO_TARGET_DIR` | `target` under repo root | Override Cargo output location. Use a short absolute path when x64-to-ARM64 builds approach Windows path-length limits. | | `Z3_LIBRARY_PATH_OVERRIDE` | unset | Directory containing an x64 system `libz3.lib`; not valid for ARM64. | | `Z3_SYS_Z3_HEADER` | unset | Full `z3.h` path required with a system Z3 library. | -| `Z3_SYS_Z3_VERSION` | `4.16.0` | Pinned official prebuilt Z3 release selected by the wrapper. | +| `Z3_SYS_Z3_VERSION` | `5.1.0` | Pinned official prebuilt Z3 release selected by the wrapper. | | `READ_ONLY_GITHUB_TOKEN` | unset | Optional token for the Z3 release lookup; GitHub Actions supplies `github.token`. | | `RUSTC_WRAPPER` | inherited | The wrapper resolves an available command to an absolute path. If it is unavailable, the wrapper warns and continues without compiler caching. | diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index 1045c15950..7f800ae0f7 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -69,7 +69,7 @@ For ARM64, verify the Visual Studio instance contains the ARM64 MSVC tools, ARM64 Spectre-mitigated libraries, Clang tools, CMake tools, and a Windows SDK. Clang supplies host-native `libclang.dll` for `bindgen` and `clang-cl.exe` for ARM64 crypto dependencies such as `aws-lc-sys`. Native and -x64-to-ARM64 builds use the official prebuilt Z3 4.16.0 static library for the +x64-to-ARM64 builds use the official prebuilt Z3 5.1.0 static library for the target architecture. An x64-to-ARM64 check/build discovers and adds host-native Ninja to `PATH`, while the crypto crates select `clang-cl`. Use a short `CARGO_TARGET_DIR` if Windows path-length limits are reached. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 809f4158d2..4213239b35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -343,7 +343,7 @@ cargo build -p openshell-prover-cli --features bundled-z3 For x86-64 and ARM64 Windows MSVC builds, use one of these Z3 paths: - Prebuilt Z3 (the default for `windows:*` tasks): `z3-sys` downloads the - pinned Z3 4.16.0 GitHub release for the target architecture on the first + pinned Z3 5.1.0 GitHub release for the target architecture on the first build. Cargo reuses the extracted archive from its target directory. Windows CI authenticates the GitHub API request with `READ_ONLY_GITHUB_TOKEN` and preserves the archive in the architecture-specific Cargo target cache. For @@ -381,8 +381,8 @@ To use a local x64 Z3 release instead of the prebuilt download, set `Z3_LIBRARY_PATH_OVERRIDE` and `Z3_SYS_Z3_HEADER` before running the task: ```powershell -$env:Z3_LIBRARY_PATH_OVERRIDE='C:\path\to\z3-4.16.0-x64-win\bin' -$env:Z3_SYS_Z3_HEADER='C:\path\to\z3-4.16.0-x64-win\include\z3.h' +$env:Z3_LIBRARY_PATH_OVERRIDE='C:\path\to\z3-5.1.0-x64-win\bin' +$env:Z3_SYS_Z3_HEADER='C:\path\to\z3-5.1.0-x64-win\include\z3.h' mise run --skip-tools windows:build:x64 ``` diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index 9093a76a52..aaf8b1a8fe 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -120,7 +120,7 @@ and the locked Rolldown version. Go tests retain race coverage wherever the toolchain supports it; POSIX permission-bit checks are not Windows ACL tests. Test tasks require the Rust target architecture to match the Windows host, so an ARM64 test result is native coverage rather than x64 emulation coverage. -By default it enables the `z3-sys` prebuilt-release feature and pins Z3 4.16.0. +By default it enables the `z3-sys` prebuilt-release feature and pins Z3 5.1.0. On a clean target directory, `z3-sys` downloads the official static library for the selected Windows architecture instead of compiling Z3 through CMake/MSBuild. GitHub Actions supplies its read-only workflow token for the diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index 51179fb1e0..192c505e8f 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -55,7 +55,7 @@ $WindowsClippyPackageExcludes = $UnsupportedDriverPackageExcludes $WindowsClippyLintArgs = "-D warnings -A dead-code -A unused-imports -A clippy::unused-async" $PrebuiltZ3WorkspaceFeatures = "--features openshell-prover/prebuilt-z3" $PrebuiltZ3ServerFeatures = "--features openshell-server/prebuilt-z3,openshell-prover/prebuilt-z3" -$PrebuiltZ3Version = "4.16.0" +$PrebuiltZ3Version = "5.1.0" $Z3WorkspaceFeatures = $PrebuiltZ3WorkspaceFeatures $Z3ServerFeatures = $PrebuiltZ3ServerFeatures From 470a34635db7d2615d5cae5c61804dcbdec04eeb Mon Sep 17 00:00:00 2001 From: Artem Lytvyn Date: Tue, 22 Sep 2026 19:45:55 +0000 Subject: [PATCH 10/10] fix(api): make WatchSandbox loss-aware and resumable (#3209) * fix(api): emit warning on WatchSandbox broadcast lag instead of terminating Broadcast lag on the status, log, and platform receivers was converted to a RESOURCE_EXHAUSTED status that terminated the whole watch stream. Lag is recoverable: the receiver resumes at the oldest surviving message. Emit a SandboxStreamWarning and continue streaming instead; keep terminating on Closed. Add helpers and unit tests covering the warning payload and receiver recovery after lag. Partially addresses #3055 (cursor/resume follow up separately). Signed-off-by: Artem Lytvyn * refactor(server): group per-sandbox log bus state and stamp sequence numbers Signed-off-by: Artem Lytvyn * feat(proto): add resume cursor fields to sandbox watch API Signed-off-by: Artem Lytvyn * feat(server): stamp watch cursors from a shared per-sandbox sequence Allocate cursors from a single SeqAllocator shared by the log and platform event buses, so a sandbox's merged watch stream carries unique, strictly increasing cursors. A single resume_after_cursor can then unambiguously locate a client's position across both sources. Rewrite both publish paths to allocate the sequence, stamp event.cursor, send, and append to the tail under one lock. This removes the previous get_mut().expect() TOCTOU race where a concurrent remove() between the two lock sections could panic. Signed-off-by: Artem Lytvyn * feat(server): serve WatchSandbox resume from cursor with gap detection Add tail_after() to the log and platform event buses, returning every buffered event newer than a client's resume cursor. Each PerSandbox now tracks last_trimmed_seq (the highest seq it has evicted) so a resume is reported as an unrecoverable ResumeGap only when this bus dropped an event the client still needs. Judging gaps by evictions, not by the tail's oldest seq, is required under the shared cursor space: each bus's tail is non-contiguous in the global sequence because the other bus owns the missing seqs, so comparing against tail.front() would flag false gaps. Signed-off-by: Artem Lytvyn * feat(server): resume WatchSandbox from cursor across log and platform buses Wire resume_after_cursor into the watch producer. On a non-zero cursor, replay events strictly after it from both the log and platform buses, merge by shared cursor, and emit in order before entering the live loop. A trimmed range on either bus is an unrecoverable gap and terminates the stream with OUT_OF_RANGE carrying the requested and earliest-available cursors, distinct from recoverable lag which warns and continues. Signed-off-by: Artem Lytvyn * test(server): cover WatchSandbox cursor resume paths Add handler-level tests for the resumable watch stream: replay strictly after the client cursor, merge log and platform events in shared-cursor order, suppress duplicates when resuming at the latest cursor, and terminate with OUT_OF_RANGE when the requested cursor has been trimmed. Signed-off-by: Artem Lytvyn * docs(api): document WatchSandbox loss-awareness and resume Signed-off-by: Artem Lytvyn * fix(server): deliver watch events once and harden cursor teardown Signed-off-by: Artem Lytvyn * feat(sdk): add loss-aware resumable watch_logs to Rust SDK client Signed-off-by: Artem Lytvyn * fix(server): keep watch cursors monotonic across teardown and restart Signed-off-by: Artem Lytvyn * fix(server): merge live watch sources by cursor before emission Signed-off-by: Artem Lytvyn * fix(api): bind watch cursors to a cursor space and merge tail sources Signed-off-by: Artem Lytvyn * fix(server): revalidate the watch cursor space after collecting replay Signed-off-by: Artem Lytvyn * test(server): update the public RPC schema fingerprint for the string cursor Signed-off-by: Artem Lytvyn * fix(server): hold watch events above the publication watermark and emit the watch lag warning before its batch Signed-off-by: Artem Lytvyn * test(server): synchronize the watch live-order test with the end of initialization Signed-off-by: Artem Lytvyn * fix(sdk): use canonical sandbox name in watch_logs Signed-off-by: Artem Lytvyn * test(sdk): guard canonical-name addressing in watch_logs Signed-off-by: Artem Lytvyn * fix(server): fix public rpc schema Signed-off-by: Artem Lytvyn * fix(api): reconcile watch resume rebase Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * fix(server): bound interactive relay cleanup Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --------- Signed-off-by: Artem Lytvyn Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> Co-authored-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- Cargo.lock | 1 + architecture/gateway.md | 76 + crates/openshell-cli/src/run.rs | 3 + .../sandbox_create_lifecycle_integration.rs | 13 + crates/openshell-sdk/Cargo.toml | 1 + crates/openshell-sdk/src/client.rs | 205 ++- crates/openshell-sdk/src/error.rs | 16 + crates/openshell-sdk/src/lib.rs | 9 +- crates/openshell-sdk/src/types.rs | 103 ++ crates/openshell-sdk/tests/client_mock.rs | 397 ++++- crates/openshell-server/src/compute/mod.rs | 5 +- crates/openshell-server/src/grpc/policy.rs | 2 +- crates/openshell-server/src/grpc/sandbox.rs | 1420 ++++++++++++++++- crates/openshell-server/src/lib.rs | 1 + crates/openshell-server/src/sandbox_watch.rs | 71 +- crates/openshell-server/src/storage_proto.rs | 9 +- crates/openshell-server/src/tracing_bus.rs | 672 +++++++- crates/openshell-server/src/watch_cursor.rs | 206 +++ docs/observability/accessing-logs.mdx | 19 + proto/openshell.proto | 40 +- sdk/go/proto/openshellv1/openshell.pb.go | 69 +- 21 files changed, 3180 insertions(+), 158 deletions(-) create mode 100644 crates/openshell-server/src/watch_cursor.rs diff --git a/Cargo.lock b/Cargo.lock index 96d6617bb8..7c16585909 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4613,6 +4613,7 @@ dependencies = [ name = "openshell-sdk" version = "0.0.0" dependencies = [ + "async-stream", "async-trait", "futures", "hyper", diff --git a/architecture/gateway.md b/architecture/gateway.md index 742fa5e189..747d328921 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -440,6 +440,82 @@ Domain objects use shared metadata: stable server-generated IDs, human-readable names, creation timestamps, and labels. Crate-level details live in `crates/openshell-core/README.md`. +### Watch streams + +`WatchSandbox` merges three per-sandbox sources into one client stream: status +snapshots, server/sandbox logs, and platform events. Logs and platform events +are resumable; a shared per-sandbox allocator stamps each with a `cursor`. +Cursor-ordered delivery is guaranteed for the replay phase: on resume the +buffered events from both sources are sorted before emission. Live events are +monotonic within each source, but the two sources are read independently, so a +client should order across sources by `cursor` rather than by arrival. Status +snapshots and warnings are re-read on demand and carry an empty cursor. + +#### Cursor spaces + +A sandbox's cursors live in a **cursor space**: a `{epoch, seq}` pair, where the +epoch is a UUID minted on the first publish and `seq` counts from 1. The epoch +is dropped by `TracingLogBus::remove`, so a teardown — or a gateway restart — +retires the space, and the next publish mints a new one. A sequence number alone +cannot distinguish a caught-up client from one holding a cursor out of a space +that no longer exists, because the replacement space reuses the same numbers; +the epoch answers *which counter issued this*, which is the question resume +validation actually has to ask. Behind multiple replicas the same rule makes a +reconnect to a different replica fail loudly rather than return the wrong +events. + +On the wire a cursor is an opaque, fixed-width token. Clients may only compare +two cursors from one stream and keep the greater; the encoding zero-pads `seq` +so that byte-wise comparison matches sequence order, which is what lets every +SDK track a high-water mark without parsing. A stream only ever observes one +epoch — a reset closes both resumable broadcast receivers, ending the stream +rather than switching spaces mid-flight — so that comparison is always well +defined where clients are allowed to use it. The gateway does not rely on it: +server-side ordering runs on the raw `u64` seq carried alongside each event in +`CursoredEvent`, never on the token. + +The gateway holds a bounded in-memory tail per sandbox. Loss is reported with +two distinct, documented behaviors: + +- **Recoverable lag** — a broadcast receiver falls behind and the server skips + ahead. The stream emits a `SandboxStreamWarning` event and continues. Since + cursors are opaque, the warning is the client's only signal. +- **Unrecoverable gap** — the server sends a snapshot, then terminates with + `OUT_OF_RANGE`. Three cases reach it: the tail was trimmed past the requested + cursor, the cursor's epoch does not match the sandbox's current space, or no + space exists because nothing has been published since teardown. The status + tells the client to restart with an empty cursor; retrying the same token + fails identically. A token the gateway could not have issued is rejected + earlier, as `INVALID_ARGUMENT` on the call itself. + +Both resumable sources draw from one cursor space, so the server merges them by +seq before emitting rather than draining each in turn: on resume it replays only +events after the client's cursor, and without one it replays each bus's retained +tail. Either way the batch leaves in ascending cursor order. The two tails are +bounded independently (`log_tail_lines` and `event_tail`), so merging orders +whatever each bus kept; it does not align their depths. + +The broadcast receivers are subscribed before replay, so an event buffered during +initialization could appear in both replay and the live receiver; the producer +tracks the highest replayed seq and suppresses live events at or below it, so +each event is delivered once. That mark is per source. The two tails are read at +different instants and bounded independently, so one shared mark would let the +deeper source censor the shallower one — with `event_tail` unset the mark rises +to the newest buffered log while no platform event is replayed at all, and +platform events published during initialization are discarded as duplicates of a +replay that never ran. Subscribing never mints a cursor space, so a +resume against a torn-down sandbox cannot create the space its stale cursor is +then checked against. Clients track the highest observed `cursor` and pass it as +`resume_after_cursor` on reconnect. + +The epoch is validated twice on resume: once before reading the tails and again +once both are in hand, before anything is emitted. The check and each read take +their locks separately, so a teardown plus a republish can retire the validated +space and install a replacement in between; the reads would then apply the old +space's seq to the replacement's buffers, and a trimmed-range check that only +compares numbers would report no gap while skipping the replacement's lower +events. The second look ends the stream with `OUT_OF_RANGE` instead. + ## Persistence The gateway persistence layer is a protobuf object store. Domain services store diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 489db3136f..b6292f5dc9 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -809,6 +809,7 @@ pub async fn sandbox_create( since_time: None, log_sources: vec!["gateway".to_string()], log_min_level: String::new(), + resume_after_cursor: String::new(), }) .await .into_diagnostic()? @@ -3613,6 +3614,7 @@ async fn wait_for_lifecycle_phase( since_time: None, log_sources: Vec::new(), log_min_level: String::new(), + resume_after_cursor: String::new(), }) .await .into_diagnostic()? @@ -5847,6 +5849,7 @@ pub async fn sandbox_logs( .into_diagnostic()?, log_sources: source_filter, log_min_level: level.to_uppercase(), + resume_after_cursor: String::new(), }) .await .into_diagnostic()? diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index f1408493bc..0b638d5694 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -743,6 +743,7 @@ impl OpenShell for TestOpenShell { let _ = tx .send(Ok(SandboxStreamEvent { payload: Some(sandbox_stream_event::Payload::Sandbox(provisioning)), + cursor: String::new(), })) .await; if terminal_after_provisional_container_exit @@ -753,6 +754,7 @@ impl OpenShell for TestOpenShell { payload: Some(sandbox_stream_event::Payload::Sandbox( provisional_container_exit, )), + cursor: String::new(), })) .await; provisional_container_exit_sent.notify_waiters(); @@ -764,6 +766,7 @@ impl OpenShell for TestOpenShell { let _ = tx .send(Ok(SandboxStreamEvent { payload: Some(sandbox_stream_event::Payload::Sandbox(completed)), + cursor: String::new(), })) .await; return; @@ -777,11 +780,13 @@ impl OpenShell for TestOpenShell { message: "Started VM launcher".to_string(), ..PlatformEvent::default() })), + cursor: String::new(), })) .await; let _ = tx .send(Ok(SandboxStreamEvent { payload: Some(sandbox_stream_event::Payload::Sandbox(error)), + cursor: String::new(), })) .await; tokio::time::sleep(Duration::from_secs(5)).await; @@ -801,12 +806,14 @@ impl OpenShell for TestOpenShell { source: "gateway".to_string(), fields: HashMap::new(), })), + cursor: String::new(), })) .await; } let _ = tx .send(Ok(SandboxStreamEvent { payload: Some(sandbox_stream_event::Payload::Sandbox(ready)), + cursor: String::new(), })) .await; return; @@ -815,6 +822,7 @@ impl OpenShell for TestOpenShell { let _ = tx .send(Ok(SandboxStreamEvent { payload: Some(sandbox_stream_event::Payload::Sandbox(completed)), + cursor: String::new(), })) .await; return; @@ -829,6 +837,7 @@ impl OpenShell for TestOpenShell { message: "Preparing rootfs".to_string(), ..PlatformEvent::default() })), + cursor: String::new(), })) .await; tokio::time::sleep(Duration::from_millis(600)).await; @@ -840,12 +849,14 @@ impl OpenShell for TestOpenShell { message: "Formatting root disk".to_string(), ..PlatformEvent::default() })), + cursor: String::new(), })) .await; tokio::time::sleep(Duration::from_millis(600)).await; let _ = tx .send(Ok(SandboxStreamEvent { payload: Some(sandbox_stream_event::Payload::Sandbox(ready)), + cursor: String::new(), })) .await; return; @@ -857,11 +868,13 @@ impl OpenShell for TestOpenShell { message: "Sandbox scheduled".to_string(), ..PlatformEvent::default() })), + cursor: String::new(), })) .await; let _ = tx .send(Ok(SandboxStreamEvent { payload: Some(sandbox_stream_event::Payload::Sandbox(ready)), + cursor: String::new(), })) .await; }); diff --git a/crates/openshell-sdk/Cargo.toml b/crates/openshell-sdk/Cargo.toml index 8d80beaa74..abbac2a9de 100644 --- a/crates/openshell-sdk/Cargo.toml +++ b/crates/openshell-sdk/Cargo.toml @@ -28,6 +28,7 @@ tokio-tungstenite = { workspace = true } tonic = { workspace = true, features = ["tls-native-roots"] } tower = { workspace = true } tracing = { workspace = true } +async-stream = "0.3.6" [dev-dependencies] serde_json = { workspace = true } diff --git a/crates/openshell-sdk/src/client.rs b/crates/openshell-sdk/src/client.rs index b019491d6f..59b42e7997 100644 --- a/crates/openshell-sdk/src/client.rs +++ b/crates/openshell-sdk/src/client.rs @@ -14,13 +14,13 @@ use crate::error::{Result, SdkError}; use crate::pagination::{Page, Pager}; use crate::raw::AuthedGrpcClient; use crate::refresh::{RefreshedToken, TokenSource}; -use crate::transport; use crate::types::{ DeleteOptions, DeletionResult, ExecOptions, ExecResult, Health, ListOptions, SandboxPhase, SandboxRef, SandboxSpec, SandboxTemplateCreateSpec, SandboxTemplateListOptions, SandboxWorkloadTemplate, WorkspaceRef, }; -use futures::StreamExt; +use crate::{WatchEvent, WatchOptions, transport}; +use futures::{Stream, StreamExt}; use openshell_core::proto; use std::collections::HashMap; use std::future::Future; @@ -662,6 +662,124 @@ impl OpenShellClient { }) } + /// Watch a sandbox's logs and platform events with loss-aware resume. + /// + /// Reconnects transparently on transient stream errors, resuming from the + /// highest cursor already delivered. A recoverable server lag surfaces as + /// [`WatchEvent::Warning`] and the stream continues. + /// + /// [`SdkError::OutOfRange`] is terminal and deliberately not retried: it + /// means the resume point is gone (trimmed from the buffer, or issued by a + /// cursor space the gateway no longer has), so events between it and now + /// are unrecoverable. Auto-restarting from scratch would hide that loss, + /// which is exactly what this API exists to surface. Callers who accept the + /// gap can start a new watch with an empty + /// [`WatchOptions::resume_after_cursor`]; retrying the same cursor fails + /// identically. + pub fn watch_logs( + &self, + name: &str, + opts: WatchOptions, + ) -> impl Stream> + '_ { + let name = name.to_string(); + async_stream::try_stream!( + let sandbox = self.get_sandbox(&name).await?; + for await event in self.watch_logs_by_name(sandbox.name, "default".to_string(), opts) { + yield event?; + } + ) + } + + /// Shared watch loop over an already-resolved canonical sandbox name. + /// + /// Both [`OpenShellClient::watch_logs`] and + /// [`WorkspaceScopedClient::watch_logs`] resolve and re-confirm a name under + /// their own workspace, then delegate here so the reconnect/resume logic + /// lives in one place. + fn watch_logs_by_name( + &self, + sandbox_name: String, + workspace: String, + opts: WatchOptions, + ) -> impl Stream> + '_ { + async_stream::try_stream!( + let mut cursor = opts.resume_after_cursor; + let mut backoff = Duration::from_millis(100); + loop { + let request = proto::WatchSandboxRequest { + sandbox: sandbox_name.clone(), + workspace_scope: Some(proto::workspace_selector(&workspace)), + follow_status: false, + follow_logs: opts.follow_logs, + follow_events: opts.follow_events, + log_tail_lines: opts.log_tail_lines, + event_tail: opts.event_tail, + log_sources: opts.log_sources.clone(), + log_min_level: opts.log_min_level.clone().unwrap_or_default(), + resume_after_cursor: cursor.clone(), + ..Default::default() + }; + // Apply the same reconnect policy to the initial dial: `unary` + // only retries `Unauthenticated`, so a pre-stream transient + // (e.g. `Unavailable`) would otherwise exit without resuming. + let mut stream = match self + .unary(|mut grpc| { + let req = request.clone(); + async move { grpc.watch_sandbox(req).await } + }) + .await + { + Ok(stream) => stream, + // Transient — back off and redial from `cursor`. + Err(err) if is_retryable_stream(&err) => { + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(Duration::from_secs(2)); + continue; + } + // Trimmed cursor or any other error — terminal. + Err(err) => Err(err)?, + }; + let mut clean_eof = true; + while let Some(item) = stream.next().await { + match item { + Ok(event) => { + // A delivered event means the connection is healthy + // again; reset the reconnect backoff so a later drop + // retries promptly instead of at the capped delay. + backoff = Duration::from_millis(100); + if let Some(ev) = convert_event(event, &mut cursor) { + yield ev; + } + } + Err(status) => { + clean_eof = false; + let err = map_status(status); + match err { + // Terminal gap — never silently restart. + SdkError::OutOfRange { .. } => { + Err(err)?; + } + // Transient — back off and redial from `cursor`. + _ if is_retryable_stream(&err) => { + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(Duration::from_secs(2)); + } + // Anything else is terminal. + _ => { + Err(err)?; + } + } + break; + } + } + } + if clean_eof { + break; + } + } + ) + } + /// Run a unary RPC with OIDC-aware auth: refresh proactively before the /// call (if the token is near expiry) and, on an `Unauthenticated` /// response, force a refresh and retry exactly once. No-op auth behaves @@ -1118,6 +1236,34 @@ impl WorkspaceScopedClient { stderr, }) } + + /// Watch a sandbox's logs and platform events with loss-aware resume. + /// + /// Reconnects transparently on transient stream errors, resuming from the + /// highest cursor already delivered. A recoverable server lag surfaces as + /// [`WatchEvent::Warning`] and the stream continues. + /// + /// [`SdkError::OutOfRange`] is terminal and deliberately not retried: it + /// means the resume point is gone (trimmed from the buffer, or issued by a + /// cursor space the gateway no longer has), so events between it and now + /// are unrecoverable. Auto-restarting from scratch would hide that loss, + /// which is exactly what this API exists to surface. Callers who accept the + /// gap can start a new watch with an empty + /// [`WatchOptions::resume_after_cursor`]; retrying the same cursor fails + /// identically. + pub fn watch_logs( + &self, + name: &str, + opts: WatchOptions, + ) -> impl Stream> + '_ { + let name = name.to_string(); + async_stream::try_stream!( + let sandbox = self.get_sandbox(&name).await?; + for await event in self.client.watch_logs_by_name(sandbox.name, self.workspace.clone(), opts) { + yield event?; + } + ) + } } fn interceptor_from_config(config: &ClientConfig) -> Result { @@ -1275,6 +1421,61 @@ fn map_status(status: tonic::Status) -> SdkError { SdkError::from_status(status) } +/// Convert a wire watch event into the curated [`WatchEvent`], advancing +/// `cursor` for resumable payloads. +/// +/// Log and platform events carry the shared per-sandbox cursor and update it. +/// Warnings are recoverable loss notices with no cursor, so they never advance +/// it. Status snapshots and draft-policy updates are not part of the log/event +/// stream and are dropped (`None`). +/// +/// `cursor` is a high-water mark, not the last cursor seen. The gateway reads +/// the log and platform sources independently during live delivery, so arrival +/// order can differ from cursor order. Taking the max keeps the resume point +/// monotonic; assigning directly would let a later lower-cursor event rewind it +/// and replay already-delivered events after a reconnect. +/// +/// The comparison is a plain byte-wise string compare on an opaque token. That +/// is the one operation the gateway permits on a cursor, and it is well defined +/// here because both cursors come from the same stream: the encoding is fixed +/// width within a cursor space, and a stream never spans two spaces (a reset +/// ends it). Never parse the token — its layout is not part of the contract. +fn convert_event(event: proto::SandboxStreamEvent, cursor: &mut String) -> Option { + match event.payload? { + proto::sandbox_stream_event::Payload::Log(line) => { + if event.cursor > *cursor { + cursor.clone_from(&event.cursor); + } + Some(WatchEvent::Log { + line: line.into(), + cursor: event.cursor, + }) + } + proto::sandbox_stream_event::Payload::Event(platform) => { + if event.cursor > *cursor { + cursor.clone_from(&event.cursor); + } + Some(WatchEvent::Event { + event: platform.into(), + cursor: event.cursor, + }) + } + proto::sandbox_stream_event::Payload::Warning(warning) => Some(WatchEvent::Warning { + message: warning.message, + }), + proto::sandbox_stream_event::Payload::Sandbox(_) + | proto::sandbox_stream_event::Payload::DraftPolicyUpdate(_) => None, + } +} + +/// Whether a mid-stream error is a transient condition worth reconnecting on. +/// +/// Only `Unavailable` (connection drop, gateway restart) is retryable; every +/// other status is terminal so the caller surfaces it instead of looping. +fn is_retryable_stream(err: &SdkError) -> bool { + matches!(err, SdkError::Rpc { code, .. } if *code == tonic::Code::Unavailable as i32) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-sdk/src/error.rs b/crates/openshell-sdk/src/error.rs index be61bb00e9..0422f9bf8e 100644 --- a/crates/openshell-sdk/src/error.rs +++ b/crates/openshell-sdk/src/error.rs @@ -104,6 +104,19 @@ pub enum SdkError { /// Original gateway status, including unknown details and metadata. status: Box, }, + + /// Gateway could not honor a resume cursor because the requested position + /// was already trimmed from its buffer (gRPC `OutOfRange`). The stream is + /// terminated; restart observation and, if needed, read missing lines from + /// the sandbox log files. + #[error("out of range: {message}")] + #[diagnostic(code(openshell::sdk::out_of_range))] + OutOfRange { + /// Error message. + message: String, + /// Original gateway status, including details and metadata. + status: Box, + }, } impl SdkError { @@ -155,6 +168,7 @@ impl SdkError { match code { tonic::Code::NotFound => Self::NotFound { message, status }, tonic::Code::AlreadyExists => Self::AlreadyExists { message, status }, + tonic::Code::OutOfRange => Self::OutOfRange { message, status }, tonic::Code::InvalidArgument => Self::InvalidConfig { message, status: Some(status), @@ -178,6 +192,7 @@ impl SdkError { Self::InvalidConfig { status, .. } | Self::Auth { status, .. } => status.as_deref(), Self::NotFound { status, .. } | Self::AlreadyExists { status, .. } + | Self::OutOfRange { status, .. } | Self::Rpc { status, .. } => Some(status), _ => None, } @@ -228,6 +243,7 @@ impl SdkError { Self::NotFound { .. } => "not_found", Self::AlreadyExists { .. } => "already_exists", Self::Rpc { .. } => "rpc", + Self::OutOfRange { .. } => "out_of_range", } } } diff --git a/crates/openshell-sdk/src/lib.rs b/crates/openshell-sdk/src/lib.rs index eac522a187..269104ae2d 100644 --- a/crates/openshell-sdk/src/lib.rs +++ b/crates/openshell-sdk/src/lib.rs @@ -51,8 +51,9 @@ pub use pagination::{Page, Pager}; pub use refresh::{Refresh, RefreshError, RefreshedToken, TokenSource}; pub use types::{ DeleteOptions, DeletionOutcome, DeletionResult, ExecOptions, ExecResult, Health, ListOptions, - SandboxPhase, SandboxRef, SandboxResources, SandboxServiceLevel, SandboxSpec, SandboxStartup, - SandboxTemplateCreateSpec, SandboxTemplateListOptions, SandboxWorkloadConfig, - SandboxWorkloadTemplate, SandboxWorkloadTemplateProvenance, SandboxWorkloadTemplateSpec, - ServiceExposure, ServiceStatus, WorkspaceRef, + LogLine, PlatformEvent, SandboxPhase, SandboxRef, SandboxResources, SandboxServiceLevel, + SandboxSpec, SandboxStartup, SandboxTemplateCreateSpec, SandboxTemplateListOptions, + SandboxWorkloadConfig, SandboxWorkloadTemplate, SandboxWorkloadTemplateProvenance, + SandboxWorkloadTemplateSpec, ServiceExposure, ServiceStatus, WatchEvent, WatchOptions, + WorkspaceRef, }; diff --git a/crates/openshell-sdk/src/types.rs b/crates/openshell-sdk/src/types.rs index a9fd544f14..57bed05e08 100644 --- a/crates/openshell-sdk/src/types.rs +++ b/crates/openshell-sdk/src/types.rs @@ -81,6 +81,61 @@ pub enum ServiceStatus { Unhealthy, } +/// One item from a reusable sandbox stream. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum WatchEvent { + /// A server/supervisor log line. Carries an opaque resume cursor. + Log { line: LogLine, cursor: String }, + /// A platform event. Carries an opaque resume cursor. + Event { + event: PlatformEvent, + cursor: String, + }, + /// Recoverable loss — the stream continues. No cursor (empty). + Warning { message: String }, +} + +/// Options for [`crate::client::OpenShellClient::watch_logs`]. +#[derive(Debug, Clone, Default)] +pub struct WatchOptions { + pub follow_logs: bool, + pub follow_events: bool, + pub log_sources: Vec, + pub log_min_level: Option, + /// Opaque cursor to resume after. Empty starts from the tail. + /// + /// Use a cursor taken from a [`WatchEvent`] of a previous watch on the same + /// sandbox. Do not construct or parse one: the encoding is not part of the + /// gateway's contract. + pub resume_after_cursor: String, + pub log_tail_lines: u32, + pub event_tail: u32, +} + +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct LogLine { + pub sandbox_id: String, + pub timestamp_ms: i64, + pub level: String, + pub target: String, + pub message: String, + pub source: String, + pub fields: HashMap, +} + +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct PlatformEvent { + pub timestamp_ms: i64, + pub source: String, + pub r#type: String, + pub reason: String, + pub message: String, + pub metadata: HashMap, +} + impl From for ServiceStatus { fn from(value: proto::ServiceStatus) -> Self { match value { @@ -92,6 +147,54 @@ impl From for ServiceStatus { } } +impl From for LogLine { + fn from(value: proto::SandboxLogLine) -> Self { + // The wire contract treats an empty source as "gateway" for backward + // compatibility with pre-`source` producers. Normalize here so callers + // never have to special-case the empty string. + let source = if value.source.is_empty() { + "gateway".to_string() + } else { + value.source + }; + Self { + sandbox_id: value.sandbox_id, + // The wire contract carries `google.protobuf.Timestamp`; these + // curated types stay dependency-light and expose milliseconds, the + // same reduction the CLI applies at its own presentation edge. An + // absent or unrepresentable timestamp reads as 0, which is what + // this field meant before the wire types gained presence. + timestamp_ms: value + .event_time + .as_ref() + .and_then(|time| openshell_core::time::timestamp_to_millis(time).ok()) + .unwrap_or(0), + level: value.level, + target: value.target, + message: value.message, + source, + fields: value.fields, + } + } +} + +impl From for PlatformEvent { + fn from(value: proto::PlatformEvent) -> Self { + Self { + timestamp_ms: value + .event_time + .as_ref() + .and_then(|time| openshell_core::time::timestamp_to_millis(time).ok()) + .unwrap_or(0), + source: value.source, + r#type: value.r#type, + reason: value.reason, + message: value.message, + metadata: value.metadata, + } + } +} + impl From for ServiceStatus { fn from(value: i32) -> Self { proto::ServiceStatus::try_from(value).map_or(Self::Unspecified, Self::from) diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs index 4051cd5136..1c86ada7dd 100644 --- a/crates/openshell-sdk/tests/client_mock.rs +++ b/crates/openshell-sdk/tests/client_mock.rs @@ -13,7 +13,8 @@ use openshell_core::proto::open_shell_server::{OpenShell, OpenShellServer}; use openshell_sdk::{ AuthConfig, ClientConfig, ExecOptions, ListOptions, OpenShellClient, Refresh, RefreshError, RefreshedToken, SandboxPhase, SandboxSpec, SandboxTemplateCreateSpec, - SandboxTemplateListOptions, ServiceExposure, ServiceStatus as SdkServiceStatus, + SandboxTemplateListOptions, ServiceExposure, ServiceStatus as SdkServiceStatus, WatchEvent, + WatchOptions, }; use std::collections::HashMap; use std::sync::Arc; @@ -21,6 +22,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use tokio::net::TcpListener; use tokio::sync::Mutex; +use tokio_stream::StreamExt; use tokio_stream::wrappers::TcpListenerStream; use tonic::{Response, Status}; @@ -72,6 +74,25 @@ struct MockState { require_bearer: Option, /// Count of requests rejected by the `require_bearer` gate. unauth_hits: AtomicU32, + last_watch_requests: Mutex>, + watch_calls: AtomicU32, + // Per-dial script. Outer Vec index = dial number. Inner = events to send, + // then how to end that dial. + watch_script: Vec, +} + +#[derive(Debug, Clone)] +struct WatchDial { + events: Vec, + end: DialEnd, +} + +#[derive(Debug, Clone)] +enum DialEnd { + Clean, + Err(tonic::Code), + /// The dial itself fails before any stream opens (pre-stream RPC error). + FailDial(tonic::Code), } #[derive(Clone)] @@ -113,6 +134,53 @@ fn sandbox_with_phase_ws( } } +/// Encode the wire cursor for `seq`, spelled out rather than built with the +/// server's encoder. +/// +/// The SDK treats cursors as opaque, so nothing in this crate can produce one. +/// Writing the format by hand also pins it independently: the padding is what +/// makes the SDK's byte-wise high-water comparison agree with sequence order, +/// and a server-side change that dropped it would have to break this literal +/// before it could break a client. +fn test_cursor(seq: u64) -> String { + format!("v1:11111111-1111-4111-8111-111111111111:{seq:020}") +} + +fn log_event(seq: u64, msg: &str) -> proto::SandboxStreamEvent { + proto::SandboxStreamEvent { + payload: Some(proto::sandbox_stream_event::Payload::Log( + proto::SandboxLogLine { + sandbox_id: "id-my-box".into(), + event_time: None, + level: "INFO".into(), + target: "t".into(), + message: msg.into(), + source: "sandbox".into(), + fields: HashMap::new(), + }, + )), + cursor: test_cursor(seq), + } +} + +fn warning_event(msg: &str) -> proto::SandboxStreamEvent { + proto::SandboxStreamEvent { + payload: Some(proto::sandbox_stream_event::Payload::Warning( + proto::SandboxStreamWarning { + message: msg.into(), + }, + )), + cursor: String::new(), + } +} + +fn watch_opts() -> WatchOptions { + WatchOptions { + follow_logs: true, + ..Default::default() + } +} + fn workspace_proto(name: &str, phase: proto::datamodel::v1::WorkspacePhase) -> proto::Workspace { proto::Workspace { metadata: Some(proto::datamodel::v1::ObjectMeta { @@ -807,9 +875,48 @@ impl OpenShell for TestOpenShell { async fn watch_sandbox( &self, - _: tonic::Request, + request: tonic::Request, ) -> Result, Status> { - Err(Status::unimplemented("unused")) + let dial = self.state.watch_calls.fetch_add(1, Ordering::SeqCst) as usize; + let req = request.into_inner(); + + // The gateway resolves `sandbox` as a workspace-scoped canonical name, + // so an object ID here is NOT_FOUND rather than a silent no-op stream. + // Mirrored so every watch test fails on ID-addressed requests. + let resolved = self.state.last_get_name.lock().await.clone(); + if let Some(resolved) = resolved + && req.sandbox != resolved + { + return Err(Status::not_found(format!( + "sandbox '{}' not found", + req.sandbox + ))); + } + + self.state.last_watch_requests.lock().await.push(req); + + let script = self.state.watch_script.get(dial).cloned(); + if let Some(WatchDial { + end: DialEnd::FailDial(code), + .. + }) = script + { + return Err(Status::new(code, "scripted dial failure")); + } + let (tx, rx) = tokio::sync::mpsc::channel(8); + tokio::spawn(async move { + let Some(dial) = script else { return }; + for ev in dial.events { + let _ = tx.send(Ok(ev)).await; + } + if let DialEnd::Err(code) = dial.end { + let _ = tx.send(Err(Status::new(code, "scripted"))).await; + } + // tx dropped here → stream ends + }); + Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new( + rx, + ))) } async fn submit_policy_analysis( @@ -1978,3 +2085,287 @@ async fn raw_grpc_fresh_refreshes_before_raw_call() { "raw_grpc_fresh must refresh the near-expiry token exactly once" ); } + +#[tokio::test] +async fn watch_logs_forwards_logs_and_warnings() { + let state = Arc::new(MockState { + phase_sequence: vec![proto::SandboxPhase::Ready], + watch_script: vec![WatchDial { + events: vec![ + log_event(1, "a"), + warning_event("lagged"), + log_event(2, "b"), + ], + end: DialEnd::Clean, + }], + ..Default::default() + }); + let endpoint = start_mock(state.clone()).await; + let client = connect(&endpoint).await; + + let stream = client.watch_logs("my-box", watch_opts()); + tokio::pin!(stream); + + match stream.next().await.unwrap().unwrap() { + WatchEvent::Log { line, cursor } => { + assert_eq!(cursor, test_cursor(1)); + assert_eq!(line.message, "a"); + } + e => panic!("expected log, got {e:?}"), + } + + assert!(matches!( + stream.next().await.unwrap().unwrap(), + WatchEvent::Warning { .. } + )); + match stream.next().await.unwrap().unwrap() { + WatchEvent::Log { cursor, .. } => assert_eq!(cursor, test_cursor(2)), + e => panic!("expected log, got {e:?}"), + } + assert!(stream.next().await.is_none()); +} + +#[tokio::test] +async fn watch_logs_resumes_after_reconnect() { + let state = Arc::new(MockState { + phase_sequence: vec![proto::SandboxPhase::Ready], + watch_script: vec![ + WatchDial { + events: vec![log_event(1, "a"), log_event(2, "b")], + end: DialEnd::Err(tonic::Code::Unavailable), + }, + WatchDial { + events: vec![log_event(3, "c")], + end: DialEnd::Clean, + }, + ], + ..Default::default() + }); + let endpoint = start_mock(state.clone()).await; + let client = connect(&endpoint).await; + + let stream = client.watch_logs("my-box", watch_opts()); + tokio::pin!(stream); + for want in [1u64, 2, 3] { + assert!( + matches!(stream.next().await.unwrap().unwrap(), WatchEvent::Log { cursor, .. } if cursor == test_cursor(want)) + ); + } + assert!(stream.next().await.is_none()); + + let reqs = state.last_watch_requests.lock().await; + assert_eq!(reqs.len(), 2); + assert_eq!(reqs[0].resume_after_cursor, ""); + // Resumed from the highest delivered cursor, forwarded verbatim. + assert_eq!(reqs[1].resume_after_cursor, test_cursor(2)); +} + +#[tokio::test] +async fn watch_logs_resumes_from_highest_cursor_when_arrival_is_unordered() { + // The gateway reads the log and platform sources independently during live + // delivery, so arrival order can differ from cursor order. The resume point + // must be the highest cursor seen, not the last one. + let state = Arc::new(MockState { + phase_sequence: vec![proto::SandboxPhase::Ready], + watch_script: vec![ + WatchDial { + events: vec![log_event(3, "c"), log_event(1, "a")], + end: DialEnd::Err(tonic::Code::Unavailable), + }, + WatchDial { + events: vec![log_event(4, "d")], + end: DialEnd::Clean, + }, + ], + ..Default::default() + }); + let endpoint = start_mock(state.clone()).await; + let client = connect(&endpoint).await; + + let stream = client.watch_logs("my-box", watch_opts()); + tokio::pin!(stream); + for want in [3u64, 1, 4] { + assert!( + matches!(stream.next().await.unwrap().unwrap(), WatchEvent::Log { cursor, .. } if cursor == test_cursor(want)) + ); + } + assert!(stream.next().await.is_none()); + + let reqs = state.last_watch_requests.lock().await; + assert_eq!(reqs.len(), 2); + // Cursor 1 arrived last but must not rewind the resume point to 1, which + // would make the gateway replay cursors 2 and 3 all over again. + assert_eq!(reqs[1].resume_after_cursor, test_cursor(3)); +} + +#[tokio::test] +async fn watch_logs_resume_cursor_is_padded_high_water() { + // The high-water mark is a byte-wise string comparison on an opaque token, + // so it is only correct while the sequence segment is a fixed width. Seq 9 + // then seq 10 is the case that catches an unpadded encoding: "...:9" sorts + // above "...:10", so the resume point would rewind to 9 and the gateway + // would replay an event the client already has. + let state = Arc::new(MockState { + phase_sequence: vec![proto::SandboxPhase::Ready], + watch_script: vec![ + WatchDial { + events: vec![log_event(9, "i"), log_event(10, "j")], + end: DialEnd::Err(tonic::Code::Unavailable), + }, + WatchDial { + events: vec![], + end: DialEnd::Clean, + }, + ], + ..Default::default() + }); + let endpoint = start_mock(state.clone()).await; + let client = connect(&endpoint).await; + + let stream = client.watch_logs("my-box", watch_opts()); + tokio::pin!(stream); + for want in [9u64, 10] { + assert!( + matches!(stream.next().await.unwrap().unwrap(), WatchEvent::Log { cursor, .. } if cursor == test_cursor(want)) + ); + } + assert!(stream.next().await.is_none()); + + let reqs = state.last_watch_requests.lock().await; + assert_eq!(reqs.len(), 2); + assert_eq!(reqs[1].resume_after_cursor, test_cursor(10)); +} + +#[tokio::test] +async fn watch_logs_normalizes_empty_log_source_to_gateway() { + let mut event = log_event(1, "a"); + if let Some(proto::sandbox_stream_event::Payload::Log(ref mut line)) = event.payload { + line.source = String::new(); + } + + let state = Arc::new(MockState { + phase_sequence: vec![proto::SandboxPhase::Ready], + watch_script: vec![WatchDial { + events: vec![event], + end: DialEnd::Clean, + }], + ..Default::default() + }); + let endpoint = start_mock(state.clone()).await; + let client = connect(&endpoint).await; + + let stream = client.watch_logs("my-box", watch_opts()); + tokio::pin!(stream); + let WatchEvent::Log { line, .. } = stream.next().await.unwrap().unwrap() else { + panic!("expected a log event"); + }; + // The wire contract treats an omitted source as "gateway". + assert_eq!(line.source, "gateway"); +} + +#[tokio::test] +async fn watch_logs_retries_initial_dial_failure() { + let state = Arc::new(MockState { + phase_sequence: vec![proto::SandboxPhase::Ready], + watch_script: vec![ + // First dial fails before the stream opens. + WatchDial { + events: vec![], + end: DialEnd::FailDial(tonic::Code::Unavailable), + }, + // Second dial succeeds and delivers. + WatchDial { + events: vec![log_event(1, "a")], + end: DialEnd::Clean, + }, + ], + ..Default::default() + }); + let endpoint = start_mock(state.clone()).await; + let client = connect(&endpoint).await; + + let stream = client.watch_logs("my-box", watch_opts()); + tokio::pin!(stream); + assert!(matches!( + stream.next().await.unwrap().unwrap(), + WatchEvent::Log { ref cursor, .. } if *cursor == test_cursor(1) + )); + assert!(stream.next().await.is_none()); + + let reqs = state.last_watch_requests.lock().await; + assert_eq!(reqs.len(), 2); // dialed twice: failed, then reconnected + assert_eq!(reqs[0].resume_after_cursor, ""); + assert_eq!(reqs[1].resume_after_cursor, ""); // nothing delivered yet on retry +} + +#[tokio::test] +async fn watch_logs_gap_terminates_out_of_range() { + let state = Arc::new(MockState { + phase_sequence: vec![proto::SandboxPhase::Ready], + watch_script: vec![WatchDial { + events: vec![log_event(1, "a")], + end: DialEnd::Err(tonic::Code::OutOfRange), + }], + ..Default::default() + }); + let endpoint = start_mock(state.clone()).await; + let client = connect(&endpoint).await; + + let stream = client.watch_logs("my-box", watch_opts()); + tokio::pin!(stream); + assert!(matches!( + stream.next().await.unwrap().unwrap(), + WatchEvent::Log { ref cursor, .. } if *cursor == test_cursor(1) + )); + let err = stream.next().await.unwrap().unwrap_err(); + assert_eq!(err.code(), "out_of_range"); + assert!(stream.next().await.is_none()); + + // No redial. OUT_OF_RANGE is terminal for every cause the gateway uses it + // for -- a trimmed cursor or one from a retired cursor space -- because + // reconnecting would silently paper over events that are already lost. + assert_eq!(state.last_watch_requests.lock().await.len(), 1); +} + +// `sandbox` addresses a workspace-scoped canonical name, and the mock's +// `id-{name}` ids differ from the names, so sending a resolved object id here +// reaches the gateway as NOT_FOUND and the watch never streams. +#[tokio::test] +async fn watch_logs_addresses_the_sandbox_by_canonical_name() { + for workspace in ["default", "production"] { + let state = Arc::new(MockState { + phase_sequence: vec![proto::SandboxPhase::Ready], + watch_script: vec![WatchDial { + events: vec![log_event(1, "a")], + end: DialEnd::Clean, + }], + ..Default::default() + }); + let endpoint = start_mock(state.clone()).await; + let client = connect(&endpoint).await; + + let scoped = client.workspace(workspace); + let mut stream: std::pin::Pin>> = + if workspace == "default" { + Box::pin(client.watch_logs("watched", watch_opts())) + } else { + Box::pin(scoped.watch_logs("watched", watch_opts())) + }; + + assert!( + matches!( + stream.next().await.unwrap().unwrap(), + WatchEvent::Log { ref cursor, .. } if *cursor == test_cursor(1) + ), + "{workspace}: first event must stream" + ); + + let reqs = state.last_watch_requests.lock().await; + assert_eq!(reqs.len(), 1, "{workspace}"); + assert_eq!(reqs[0].sandbox, "watched", "{workspace}: name, not id"); + assert_eq!( + selected_workspace(&reqs[0].workspace_scope), + Some(workspace) + ); + } +} diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 9f32743137..ad98e46247 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -3591,6 +3591,8 @@ impl ComputeRuntime { public_platform_event_from_driver(&event), ), ), + // Placeholder: platform_event_bus.publish() stamps the cursor. + cursor: String::new(), }, ); } @@ -4399,8 +4401,9 @@ impl ComputeRuntime { } fn cleanup_sandbox_state(&self, sandbox_id: &str) { + // `tracing_log_bus.remove` also clears the platform event bus and resets + // the shared cursor allocator last (see its docs). self.tracing_log_bus.remove(sandbox_id); - self.tracing_log_bus.platform_event_bus.remove(sandbox_id); self.sandbox_watch_bus.remove(sandbox_id); } diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 27fcf946da..4d7c52853d 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -4712,7 +4712,7 @@ pub(super) async fn handle_get_sandbox_logs( .into_iter() .filter_map(|evt| { if let Some(openshell_core::proto::sandbox_stream_event::Payload::Log(log)) = - evt.payload + evt.event.payload { if let Some(since_time) = since_time.as_ref() { let event_time = log.event_time.as_ref()?; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index b75f464f0b..fd83e008e3 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -18,6 +18,8 @@ use crate::pagination::Pagination; use crate::persistence::{ ObjectLabels, ObjectListQuery, ObjectType, WriteCondition, generate_name, }; +use crate::tracing_bus::CursoredEvent; +use crate::watch_cursor::WatchCursor; use futures::future; use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::proto::datamodel::v1::ObjectMeta; @@ -53,7 +55,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::task::{Context, Poll}; use tokio::net::{TcpListener, TcpStream}; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{broadcast, mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; use tracing::{debug, info, warn}; @@ -81,6 +83,32 @@ const MAX_CREATE_SERVICE_EXPOSURES: usize = 32; #[path = "interactive_exec_tests.rs"] mod interactive_exec_tests; +/// Terminal status for a resume cursor issued by a cursor space that is gone. +/// +/// Retrying the same token fails identically, so the guidance has to be +/// "restart without one" -- otherwise an SDK that reconnects on `OUT_OF_RANGE` +/// spins. The token is not echoed back. +const RESUME_SPACE_GONE: &str = "resume_after_cursor belongs to a cursor space that no longer \ + exists; the gateway restarted or the sandbox's buffers were torn down. Restart the watch \ + with an empty resume_after_cursor. Events published in the meantime are not recoverable."; + +/// Terminal status for a resume cursor ahead of everything its space issued. +const RESUME_CURSOR_AHEAD: &str = "resume_after_cursor is ahead of every cursor this sandbox has \ + issued. Restart the watch with an empty resume_after_cursor."; + +/// Whether `sandbox_id`'s cursor space is still the one that issued `epoch`. +/// +/// A teardown retires the space and the next publish mints a replacement that +/// renumbers from 1, so a surviving epoch is the only proof that a seq validated +/// earlier still addresses the same numbering. Absent counts as changed: there +/// is nothing left for the cursor to point into. +fn cursor_space_is(state: &ServerState, sandbox_id: &str, epoch: uuid::Uuid) -> bool { + state + .tracing_log_bus + .cursor_space(sandbox_id) + .is_some_and(|space| space.epoch == epoch) +} + #[derive(Debug)] pub struct WatchSandboxStream { receiver: ReceiverStream>, @@ -1752,6 +1780,19 @@ pub(super) async fn handle_watch_sandbox( let log_min_level = req.log_min_level; let event_tail = req.event_tail; + // Decode the resume cursor before spawning the producer. A token this + // server could not have issued is pure input validation, in the same class + // as the `id is required` check above, so it fails the RPC rather than + // arriving as the first item of an otherwise-established stream. + let resume_after = if req.resume_after_cursor.is_empty() { + None + } else { + Some( + WatchCursor::parse(&req.resume_after_cursor) + .map_err(|e| Status::invalid_argument(e.to_string()))?, + ) + }; + let (tx, rx) = mpsc::channel::>(256); let state = state.clone(); @@ -1808,6 +1849,8 @@ pub(super) async fn handle_watch_sandbox( sandbox.clone(), ), ), + // Status snapshots are re-read, not resumed by cursor. + cursor: String::new(), })) .await; @@ -1831,13 +1874,142 @@ pub(super) async fn handle_watch_sandbox( } } - // Replay tail logs (best-effort), filtered by log_since_time and log_sources. - if follow_logs { - for evt in state.tracing_log_bus.tail(&sandbox_id, log_tail as usize) { - if let Some(openshell_core::proto::sandbox_stream_event::Payload::Log( - ref log, - )) = evt.payload - { + // Highest seq the tail/replay phase already handled, tracked per + // source. The broadcast receivers were subscribed before replay ran, + // so an event published during initialization can sit in both the + // replay buffer and a live receiver; the live loop suppresses events + // at or below its source's mark so each is delivered exactly once. + // + // The two marks must stay separate. Both buses number from one + // shared cursor space, but they are read at different instants and + // bounded independently (`log_tail_lines` vs `event_tail`, which has + // no default and so replays nothing unless the client asks). A + // single shared mark therefore lets the deeper source censor the + // shallower one: with the default `event_tail` of 0 the mark rises + // to the newest buffered log while no platform event was replayed at + // all, and every platform event published in the initialization + // window is dropped as a duplicate of something never sent. Keyed by + // source, an event is suppressed only if its own source's replay + // actually covered it. + // + // No unit test pins this. The only reachable window is between the + // subscribe above and the log tail read below -- an event published + // earlier is replayed rather than live, and one published later + // outranks the mark -- and the producer crosses that window with no + // await a test can wedge open. Reproducing it needs a seam in the + // producer, which is not worth adding to production code. + let resume_seq = resume_after.map_or(0, |resume| resume.seq); + let mut log_cutoff: u64 = resume_seq; + let mut platform_cutoff: u64 = resume_seq; + + if let Some(resume) = resume_after { + // Resume: replay events strictly after the client's cursor from both + // resumable buses. Either bus reporting a trimmed range is an + // unrecoverable gap -> terminate with a documented status. + use openshell_core::proto::sandbox_stream_event::Payload; + + // A cursor is only a position inside the space that issued it. + // A gateway restart, a bus teardown, or a reconnect landing on + // another replica starts a new space numbered from 1 with no + // record of the cursors the old one handed out. The buses then + // look merely empty, so `tail_after` reports no gap -- treating + // that as "caught up" would pin the cutoff to a stale number + // and silently swallow every live event beneath it. Comparing + // epochs answers "did this cursor come from *this* space?", + // which no numeric bound can. + match state.tracing_log_bus.cursor_space(&sandbox_id) { + None => { + let _ = tx.send(Err(Status::out_of_range(RESUME_SPACE_GONE))).await; + return; + } + Some(space) if space.epoch != resume.epoch => { + let _ = tx.send(Err(Status::out_of_range(RESUME_SPACE_GONE))).await; + return; + } + // Right space, but ahead of anything it issued: only a + // fabricated token gets here. Reject rather than accept a + // cutoff no event can ever exceed. + Some(space) if resume.seq > space.highest_seq => { + let _ = tx + .send(Err(Status::out_of_range(RESUME_CURSOR_AHEAD))) + .await; + return; + } + Some(_) => {} + } + + let log_replay = if follow_logs { + Some(state.tracing_log_bus.tail_after(&sandbox_id, resume.seq)) + } else { + None + }; + + let platform_replay = if follow_events { + Some( + state + .tracing_log_bus + .platform_event_bus + .tail_after(&sandbox_id, resume.seq), + ) + } else { + None + }; + + // Re-check the epoch now that both tails are in hand. The check + // above and each `tail_after` take their locks independently, so + // a teardown plus a republish can retire the validated space and + // install a replacement in between. The reads would then have + // applied the old space's seq to the new space's buffers, and + // `tail_after` -- which only knows numbers -- would report no gap + // while skipping every replacement event at or below it. The + // second look is cheap and runs before anything is emitted, so a + // space that moved under us ends the stream instead of serving a + // truncated replay. + // + // Ordering is unchanged: this takes only the allocator lock and + // releases it, never held across a bus lock. + if !cursor_space_is(&state, &sandbox_id, resume.epoch) { + let _ = tx.send(Err(Status::out_of_range(RESUME_SPACE_GONE))).await; + return; + } + + // Gap check FIRST (borrows), before the merge moves the vecs. + for replay in [&log_replay, &platform_replay] { + if let Some(Err(gap)) = replay { + let _ = tx.send(Err(Status::out_of_range(format!( + "resume cursor {} is no longer available; earliest resumable cursor is {}", + gap.requested_after, gap.oldest_available + )))) + .await; + return; + } + } + + // Merge both buses by shared seq, then emit ascending. Each + // source's mark comes from its own replay -- `tail_after` + // returns ascending, so its last entry is that source's high + // water. Marking every event the phase examined, not only the + // ones that survived the filters, keeps a filtered event's live + // duplicate suppressed: the live loop does not re-apply + // `log_since_time` and would otherwise let it through. + let mut merged: Vec = Vec::new(); + if let Some(Ok(v)) = log_replay { + if let Some(last) = v.last() { + log_cutoff = log_cutoff.max(last.seq); + } + merged.extend(v); + } + if let Some(Ok(v)) = platform_replay { + if let Some(last) = v.last() { + platform_cutoff = platform_cutoff.max(last.seq); + } + merged.extend(v); + } + + merged.sort_by_key(|c| c.seq); + + for cursored in merged { + if let Some(Payload::Log(ref log)) = cursored.event.payload { if let Some(since_time) = log_since_time.as_ref() { let Some(event_time) = log.event_time.as_ref() else { continue; @@ -1858,27 +2030,88 @@ pub(super) async fn handle_watch_sandbox( continue; } } - if tx.send(Ok(evt)).await.is_err() { + if tx.send(Ok(cursored.event)).await.is_err() { return; } } - } + } else { + // Initial tail, best-effort. Both buses draw from one shared + // cursor space, so draining each in its own pass would order the + // tail by source and only incidentally by cursor: every buffered + // log would precede every buffered platform event regardless of + // which was published first. Collect both windows, sort by the + // shared seq, and emit one ascending run -- the same shape the + // resume path above uses, and the order a client comparing + // cursors expects. + // + // The two windows are truncated independently (log_tail vs + // event_tail), so this merges whatever each bus retained; it + // does not align their depths. + let mut tail: Vec = Vec::new(); + if follow_logs { + let logs = state.tracing_log_bus.tail(&sandbox_id, log_tail as usize); + if let Some(last) = logs.last() { + log_cutoff = log_cutoff.max(last.seq); + } + tail.extend(logs); + } + if follow_events { + let events = state + .tracing_log_bus + .platform_event_bus + .tail(&sandbox_id, event_tail as usize); + if let Some(last) = events.last() { + platform_cutoff = platform_cutoff.max(last.seq); + } + tail.extend(events); + } - // Replay buffered platform events. - if follow_events { - for evt in state - .tracing_log_bus - .platform_event_bus - .tail(&sandbox_id, event_tail as usize) - { - if tx.send(Ok(evt)).await.is_err() { + tail.sort_by_key(|cursored| cursored.seq); + + for cursored in tail { + // Log filters; platform events carry no log fields and pass. + if let Some(openshell_core::proto::sandbox_stream_event::Payload::Log( + ref log, + )) = cursored.event.payload + { + if let Some(since_time) = log_since_time.as_ref() { + let Some(event_time) = log.event_time.as_ref() else { + continue; + }; + let Ok(ordering) = + openshell_core::time::compare_timestamps(event_time, since_time) + else { + continue; + }; + if ordering == std::cmp::Ordering::Less { + continue; + } + } + if !log_sources.is_empty() && !source_matches(&log.source, &log_sources) { + continue; + } + if !level_matches(&log.level, &log_min_level) { + continue; + } + } + if tx.send(Ok(cursored.event)).await.is_err() { return; } } } + // Events drained above the publication watermark. They are held + // back rather than emitted so the client's highest delivered cursor + // is never above an event still queued on the other source. + let mut deferred: Vec = Vec::new(); + loop { - tokio::select! { + // Events withheld by the previous round are already in hand, so + // this round must not block on a new publication: the watermark + // that covers them has already advanced past them, and waiting + // for unrelated traffic would stall their delivery indefinitely. + let first = if deferred.is_empty() { + Some(tokio::select! { () = tx.closed() => { return; } @@ -1893,7 +2126,7 @@ pub(super) async fn handle_watch_sandbox( match state.store.get_message::(&sandbox_id).await { Ok(Some(sandbox)) => { state.sandbox_index.update_from_sandbox(&sandbox); - if tx.send(Ok(SandboxStreamEvent { payload: Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(sandbox.clone()))})).await.is_err() { + if tx.send(Ok(SandboxStreamEvent { payload: Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(sandbox.clone())), cursor: String::new() })).await.is_err() { return; } if stop_on_terminal { @@ -1912,57 +2145,162 @@ pub(super) async fn handle_watch_sandbox( } } } - Err(err) => { - let _ = tx.send(Err(crate::sandbox_watch::broadcast_to_status(err))).await; + Err(broadcast::error::RecvError::Lagged(n)) => { + // Lag is recoverable: surface a warning and keep streaming. + if tx.send(Ok(crate::sandbox_watch::lag_warning_event(n))).await.is_err() { + return; + } + } + Err(broadcast::error::RecvError::Closed) => { + let _ = tx.send(Err(Status::cancelled("stream closed"))).await; return; } } + // Status snapshots carry cursor 0 and are outside the + // resumable cursor space, so they never join a batch. + continue; } + // Both resumable sources feed one cursor space, so neither + // can be emitted on its own: `select!` picks an arbitrary + // ready branch, which would emit a higher cursor ahead of a + // lower one waiting on the other source. Take whichever woke + // us as the start of a batch and merge below. res = async { match log_rx.as_mut() { Some(rx) => rx.recv().await, None => future::pending().await, } - } => { - match res { - Ok(evt) => { - if let Some(openshell_core::proto::sandbox_stream_event::Payload::Log(ref log)) = evt.payload { - if !log_sources.is_empty() && !source_matches(&log.source, &log_sources) { - continue; - } - if !level_matches(&log.level, &log_min_level) { - continue; - } - } - if tx.send(Ok(evt)).await.is_err() { - return; - } - } - Err(err) => { - let _ = tx.send(Err(crate::sandbox_watch::broadcast_to_status(err))).await; - return; - } - } - } + } => res, res = async { match platform_rx.as_mut() { Some(rx) => rx.recv().await, None => future::pending().await, } - } => { - match res { - Ok(evt) => { - if tx.send(Ok(evt)).await.is_err() { - return; - } - } - Err(err) => { - let _ = tx.send(Err(crate::sandbox_watch::broadcast_to_status(err))).await; - return; + } => res, + }) + } else { + None + }; + + let mut batch = std::mem::take(&mut deferred); + match first { + None => {} + Some(Ok(evt)) => batch.push(evt), + Some(Err(broadcast::error::RecvError::Lagged(n))) => { + // Lag is recoverable: surface a warning and keep streaming. + if tx + .send(Ok(crate::sandbox_watch::lag_warning_event(n))) + .await + .is_err() + { + return; + } + // Carry any withheld events into the next round rather + // than dropping them with this batch. + deferred = batch; + continue; + } + Some(Err(broadcast::error::RecvError::Closed)) => { + let _ = tx.send(Err(Status::cancelled("stream closed"))).await; + return; + } + } + + // Read the publication watermark before draining. Publishers + // assign a sequence and push it onto their broadcast channel + // under one allocator lock, so every event at or below the + // sequence observed here has already reached its channel and + // the drain below is guaranteed to see it. Reading after the + // drain would admit a publish into the gap and defeat this. + // + // `None` means the cursor space is gone (teardown). Nothing + // further can be published into it, so nothing is withheld. + let watermark = state + .tracing_log_bus + .cursor_space(&sandbox_id) + .map_or(u64::MAX, |space| space.highest_seq); + + // Drain what is already queued on both sources. Sorting the + // batch restores cursor order across the two sources without + // waiting on either one. + let mut lagged = 0u64; + let mut closed = false; + for rx in [log_rx.as_mut(), platform_rx.as_mut()] + .into_iter() + .flatten() + { + loop { + match rx.try_recv() { + Ok(evt) => batch.push(evt), + Err(broadcast::error::TryRecvError::Empty) => break, + // Keep draining: the receiver is usable after a skip. + Err(broadcast::error::TryRecvError::Lagged(n)) => lagged += n, + Err(broadcast::error::TryRecvError::Closed) => { + closed = true; + break; } } } } + + batch.sort_by_key(|cursored| cursored.seq); + + // Withhold anything above the watermark. Such an event was + // published after the drain started, so a lower-cursor event + // from the other source may still be queued behind it. Emitting + // it now would let the client checkpoint above an event it has + // not seen, and resume past it after a disconnect. The next + // round re-reads the watermark, which by then covers these. + let held_from = batch.partition_point(|cursored| cursored.seq <= watermark); + deferred = batch.split_off(held_from); + + // Announce the gap before any event from this batch. A warning + // carries no cursor and is never replayed, so emitting it after + // the events lets a disconnect at the wrong moment strand the + // client past the gap: it would resume from a cursor above the + // skipped events having never learned they were dropped. + if lagged > 0 + && tx + .send(Ok(crate::sandbox_watch::lag_warning_event(lagged))) + .await + .is_err() + { + return; + } + + for cursored in batch { + // Skip events the tail/replay phase already handled, judged + // against the mark for this event's own source. Bus events + // always carry seq >= 1, so no sentinel is needed here: + // non-resumable events never reach this batch. + let is_log = matches!( + cursored.event.payload, + Some(openshell_core::proto::sandbox_stream_event::Payload::Log(_)) + ); + let cutoff = if is_log { log_cutoff } else { platform_cutoff }; + if cursored.seq <= cutoff { + continue; + } + if let Some(openshell_core::proto::sandbox_stream_event::Payload::Log( + ref log, + )) = cursored.event.payload + { + if !log_sources.is_empty() && !source_matches(&log.source, &log_sources) { + continue; + } + if !level_matches(&log.level, &log_min_level) { + continue; + } + } + if tx.send(Ok(cursored.event)).await.is_err() { + return; + } + } + + if closed { + let _ = tx.send(Err(Status::cancelled("stream closed"))).await; + return; + } } }, request_span, @@ -2882,7 +3220,7 @@ async fn stream_exec_over_relay( )), })) .await; - let _ = proxy_task.await; + finish_interactive_exec_proxy(proxy_task).await; return Ok(()); } } else { @@ -2892,12 +3230,12 @@ async fn stream_exec_over_relay( let exit_code = match exec_result { Ok(code) => code, Err(status) => { - let _ = proxy_task.await; + finish_interactive_exec_proxy(proxy_task).await; return Err(status); } }; - let _ = proxy_task.await; + finish_interactive_exec_proxy(proxy_task).await; let _ = tx .send(Ok(ExecSandboxEvent { @@ -3960,6 +4298,976 @@ mod tests { ); } + /// Seed `n` log lines onto the log bus; cursors run 1..=n. + fn seed_log_lines(state: &ServerState, sandbox_id: &str, n: usize) { + for i in 0..n { + state + .tracing_log_bus + .publish_external(openshell_core::proto::SandboxLogLine { + sandbox_id: sandbox_id.to_string(), + event_time: openshell_core::time::timestamp_from_millis(i as i64).ok(), + level: "INFO".to_string(), + target: "test".to_string(), + message: format!("line {i}"), + source: "gateway".to_string(), + ..Default::default() + }); + } + } + + fn seed_platform_event(state: &ServerState, sandbox_id: &str, reason: &str) { + state.tracing_log_bus.platform_event_bus.publish( + sandbox_id, + SandboxStreamEvent { + payload: Some(openshell_core::proto::sandbox_stream_event::Payload::Event( + openshell_core::proto::PlatformEvent { + event_time: openshell_core::time::timestamp_from_millis(0).ok(), + source: "test".to_string(), + r#type: "Normal".to_string(), + reason: reason.to_string(), + message: reason.to_string(), + metadata: HashMap::new(), + }, + )), + cursor: String::new(), + }, + ); + } + + /// Build the token a client would hold for `seq` in this sandbox's *current* + /// cursor space. Capture it before any teardown to model a real reconnect. + fn cursor_token(state: &ServerState, sandbox_id: &str, seq: u64) -> String { + let space = state + .tracing_log_bus + .cursor_space(sandbox_id) + .expect("cursor space exists; publish before taking a token"); + WatchCursor::new(space.epoch, seq).encode() + } + + /// A well-formed token from an epoch this server never issued. + fn foreign_cursor(seq: u64) -> String { + WatchCursor::new(uuid::Uuid::new_v4(), seq).encode() + } + + /// Sequence number carried by a delivered event. Panics on non-resumable + /// events, so a test that expects a log line cannot silently pass on a + /// snapshot. + fn seq_of(evt: &SandboxStreamEvent) -> u64 { + WatchCursor::parse(&evt.cursor) + .expect("resumable event must carry a valid cursor") + .seq + } + + #[tokio::test] + async fn resume_replays_only_events_after_cursor() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("resumed", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // Cursors 1,2,3. + seed_log_lines(&state, &id, 3); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: cursor_token(&state, &id, 1), + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + // Snapshot first (status re-read, no cursor). + let snap = stream.next().await.unwrap().unwrap(); + assert!( + snap.cursor.is_empty(), + "first event should be the status snapshot" + ); + + // Then only seqs 2 and 3; seq 1 already seen by the client. + let a = stream.next().await.unwrap().unwrap(); + let b = stream.next().await.unwrap().unwrap(); + assert_eq!(seq_of(&a), 2); + assert_eq!(seq_of(&b), 3); + } + + #[tokio::test] + async fn resume_merges_log_and_platform_events_in_cursor_order() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("merged", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // Interleave across the shared allocator: log=1, platform=2, log=3, platform=4. + seed_log_lines(&state, &id, 1); // cursor 1 + seed_platform_event(&state, &id, "e2"); // cursor 2 + state + .tracing_log_bus + .publish_external(openshell_core::proto::SandboxLogLine { + sandbox_id: id.clone(), + event_time: openshell_core::time::timestamp_from_millis(3).ok(), + level: "INFO".to_string(), + target: "test".to_string(), + message: "line 3".to_string(), + source: "gateway".to_string(), + ..Default::default() + }); // cursor 3 + seed_platform_event(&state, &id, "e4"); // cursor 4 + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + follow_events: true, + resume_after_cursor: cursor_token(&state, &id, 1), + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + + // Merged from both buses, ascending by shared seq: 2,3,4. + let mut got = Vec::new(); + for _ in 0..3 { + got.push(seq_of(&stream.next().await.unwrap().unwrap())); + } + assert_eq!(got, vec![2, 3, 4]); + } + + /// The initial tail is the resume path's twin: it draws from the same two + /// buses over the same shared cursor space, so it owes the client the same + /// ascending order. + /// + /// Before the merge, each bus was drained in its own pass and the tail came + /// out grouped by source -- logs 1,3 then platform 2,4 -- so a client + /// tracking the highest cursor saw it go backwards mid-tail. + #[tokio::test] + async fn initial_tail_merges_log_and_platform_events_in_cursor_order() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("tailmerged", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // Interleave across the shared allocator: log=1, platform=2, log=3, platform=4. + seed_log_lines(&state, &id, 1); // cursor 1 + seed_platform_event(&state, &id, "e2"); // cursor 2 + state + .tracing_log_bus + .publish_external(openshell_core::proto::SandboxLogLine { + sandbox_id: id.clone(), + event_time: openshell_core::time::timestamp_from_millis(3).ok(), + level: "INFO".to_string(), + target: "test".to_string(), + message: "line 3".to_string(), + source: "gateway".to_string(), + ..Default::default() + }); // cursor 3 + seed_platform_event(&state, &id, "e4"); // cursor 4 + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + follow_events: true, + // event_tail has no default; 0 would replay no platform events. + event_tail: 10, + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + + let mut got = Vec::new(); + for _ in 0..4 { + got.push(seq_of(&stream.next().await.unwrap().unwrap())); + } + assert_eq!(got, vec![1, 2, 3, 4]); + } + + #[tokio::test] + async fn live_delivery_orders_events_across_sources_by_cursor() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("liveorder", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + follow_events: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + // The snapshot itself only proves both subscriptions are live -- the + // replay reads come after it. But on this current-thread runtime the + // producer cannot yield between the two, so by the time the test task + // is scheduled again the producer has run through to the live loop. + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + + // Publish without awaiting in between. On the current-thread runtime + // the producer cannot interleave, so both channels hold ready events + // when it next polls -- the state where `select!` picks arbitrarily and + // would otherwise emit a log cursor ahead of a lower platform cursor. + for i in 0..5 { + seed_log_lines(&state, &id, 1); // odd cursors + seed_platform_event(&state, &id, &format!("e{i}")); // even cursors + } + + let mut got = Vec::new(); + for _ in 0..10 { + got.push(seq_of(&stream.next().await.unwrap().unwrap())); + } + assert_eq!(got, (1..=10).collect::>()); + } + + /// A reconnect resumes from the highest cursor the client saw, so live + /// delivery may never emit a cursor while a lower one is still undelivered. + /// The producer drains the log receiver before the platform one: a log line + /// published after that first drain but before the platform drain finishes + /// misses the batch, and its seq is below platform cursors the same batch + /// carries. Emitting them strands the log line -- a disconnect there resumes + /// above it, and no replay ever returns it. The publication watermark holds + /// back anything the drain cannot prove it saw in full. + /// + /// Reaching that interleaving takes a wide drain: the producer is first + /// parked on a full stream channel so a platform backlog accumulates, then + /// both sources are hammered from other worker threads while it walks that + /// backlog. Serialized against the test task the window does not exist -- + /// the drain holds no await a single-threaded test could wedge open. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn live_delivery_never_emits_a_cursor_above_an_undrained_event() { + use tokio_stream::StreamExt as _; + + /// Enough to overrun the 256-slot stream channel and park the producer. + const PARK: usize = 400; + /// Platform events queued while it is parked. The next drain walks all + /// of them, and that walk is the window the hammers below publish into. + /// Kept under the 1024-slot broadcast capacity so nothing is dropped. + const BACKLOG: usize = 600; + const HAMMER: usize = 300; + /// Hitting the window is a race, so repeat it. One round reproduced the + /// unfixed behavior in two runs of three; four caught it in ten of ten. + const ROUNDS: u64 = 4; + /// The single line published to confirm the producer finished + /// initialization before the rounds below start publishing. + const HANDSHAKE: u64 = 1; + const PER_ROUND: u64 = (PARK + BACKLOG + HAMMER * 2) as u64; + const TOTAL: u64 = PER_ROUND * ROUNDS + HANDSHAKE; + + let state = test_server_state().await; + let sandbox = test_sandbox("watermark", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + follow_events: true, + ..Default::default() + }), + ) + .await + .unwrap(); + let mut stream = response.into_inner(); + // The snapshot only proves both subscriptions are live; the producer + // sends it before reading either replay window. Publishing the rest of + // this test against that state is unsound: the log tail is capped at + // 200 by default, so a burst landing before the read is truncated, the + // rest is suppressed as already replayed, and the test stalls. + // + // One line is the handshake. Receiving it with a cursor -- replayed or + // live, either way -- proves the producer is past both tail reads, and + // one line cannot overflow any tail. Everything below is published into + // a producer known to be in the live loop. + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + seed_log_lines(&state, &id, 1); + let handshake = stream.next().await.unwrap().unwrap(); + assert_eq!( + seq_of(&handshake), + HANDSHAKE, + "handshake line should be seq 1" + ); + + let mut highest = HANDSHAKE; + let mut seen = HANDSHAKE; + let mut last_cursor = handshake.cursor; + + for round in 0..ROUNDS { + // Nothing reads during this round's setup, so the producer fills + // the stream channel and blocks part-way through this batch. + seed_log_lines(&state, &id, PARK); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + for i in 0..BACKLOG { + seed_platform_event(&state, &id, &format!("b{round}-{i}")); + } + + let log_hammer = { + let state = Arc::clone(&state); + let id = id.clone(); + tokio::spawn(async move { + for _ in 0..HAMMER { + seed_log_lines(&state, &id, 1); + } + }) + }; + let platform_hammer = { + let state = Arc::clone(&state); + let id = id.clone(); + tokio::spawn(async move { + for i in 0..HAMMER { + seed_platform_event(&state, &id, &format!("h{round}-{i}")); + } + }) + }; + + while seen < PER_ROUND * (round + 1) + HANDSHAKE { + let evt = tokio::time::timeout(std::time::Duration::from_secs(30), stream.next()) + .await + .unwrap_or_else(|_| { + panic!( + "live delivery stalled after {seen}/{TOTAL} events, highest {highest}" + ) + }) + .unwrap() + .unwrap(); + let seq = seq_of(&evt); + // Ascending delivery is what makes a highest-cursor resume safe. + // Without the watermark this trips: a log line published during + // the platform walk follows platform cursors emitted above it. + assert!(seq > highest, "cursor {seq} emitted after {highest}"); + highest = seq; + last_cursor = evt.cursor; + seen += 1; + } + + log_hammer.await.unwrap(); + platform_hammer.await.unwrap(); + } + + // Every seq in the space belongs to one of the two buses, so `TOTAL` + // ascending events ending at `TOTAL` means none was skipped. + assert_eq!(highest, TOTAL, "live delivery skipped a cursor"); + + // Reconnecting from that cursor is consistent with what was delivered: + // the replay resumes at the next event rather than past one. + seed_log_lines(&state, &id, 1); + seed_platform_event(&state, &id, "after-resume"); + drop(stream); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + follow_events: true, + resume_after_cursor: last_cursor, + ..Default::default() + }), + ) + .await + .unwrap(); + let mut stream = response.into_inner(); + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + + let a = stream.next().await.unwrap().unwrap(); + let b = stream.next().await.unwrap().unwrap(); + assert_eq!(seq_of(&a), TOTAL + 1); + assert_eq!(seq_of(&b), TOTAL + 2); + } + + /// A lag warning carries no cursor, so it is never replayed on resume. Sent + /// after the surviving events of its own batch, a disconnect in between + /// leaves the client checkpointed past the dropped range having never been + /// told anything was dropped. The warning must lead the batch. + /// + /// Which branch `select!` takes is arbitrary, so the run is repeated: the + /// assertion holds on both paths, but only the platform-first path reaches + /// the drain's lag counter, which is where the ordering used to be wrong. + /// Eight attempts caught the unfixed ordering in three runs of five; forty + /// caught it in six of six. + #[tokio::test] + async fn lag_warning_precedes_the_events_of_its_batch() { + use openshell_core::proto::sandbox_stream_event::Payload; + use tokio_stream::StreamExt as _; + + /// `select!` polls the log receiver before the platform one in three of + /// its four rotations, and only the platform-first path reaches the + /// drain's lag counter. Repeat enough that missing it is negligible. + const ATTEMPTS: usize = 40; + + let state = test_server_state().await; + + for attempt in 0..ATTEMPTS { + let sandbox = test_sandbox(&format!("lagorder{attempt}"), Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + follow_events: true, + ..Default::default() + }), + ) + .await + .unwrap(); + let mut stream = response.into_inner(); + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + + // One platform event plus a log burst past the 1024-slot broadcast + // capacity, published without an await in between so the producer + // sees both on a single wake-up: a deliverable event and a drop. + seed_platform_event(&state, &id, "e1"); + seed_log_lines(&state, &id, 1100); + + let first = stream.next().await.unwrap().unwrap(); + assert!( + matches!(first.payload, Some(Payload::Warning(_))), + "the lag warning must arrive before any event of the lagged batch, got {:?}", + first.payload + ); + } + } + + #[tokio::test] + async fn resume_at_latest_cursor_suppresses_duplicates() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("nodup", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // Cursors 1,2,3; client already saw through 3. + seed_log_lines(&state, &id, 3); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: cursor_token(&state, &id, 3), + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + + // No resumable events remain; the live loop yields nothing promptly. + let next = tokio::time::timeout(std::time::Duration::from_millis(200), stream.next()).await; + assert!( + next.is_err(), + "expected no further events after resume at latest cursor, got {next:?}" + ); + } + + #[tokio::test] + async fn resume_from_trimmed_cursor_terminates_out_of_range() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("gap", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // Exceed the 2000-line tail so the earliest cursors are trimmed. + seed_log_lines(&state, &id, 2005); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + // Seq 2 was trimmed; this is an unrecoverable gap. + resume_after_cursor: cursor_token(&state, &id, 2), + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + // Snapshot still arrives first (fresh state), then the terminal gap status. + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + + let err = stream + .next() + .await + .unwrap() + .expect_err("trimmed cursor must terminate the stream"); + assert_eq!(err.code(), tonic::Code::OutOfRange, "{err:?}"); + assert!( + err.message().contains('2'), + "gap status should report the requested cursor: {}", + err.message() + ); + + // Stream ends after the terminal status. + assert!(stream.next().await.is_none()); + } + + /// The guard behind the producer's post-replay epoch re-check. + /// + /// Validation and the two `tail_after` reads take their locks separately, so + /// a teardown plus a republish can swap the space in between and leave the + /// reads applying an old seq to a replacement's buffers -- `tail_after` only + /// compares numbers, so it reports no gap while skipping every replacement + /// event at or below that seq. The producer re-checks the epoch once both + /// tails are in hand and before emitting anything; this pins what that check + /// must answer. + /// + /// The interleaving itself is not reachable from a test: the producer runs + /// validation, both reads, and the re-check with no await in between, so + /// there is nothing to suspend it on. + #[tokio::test] + async fn cursor_space_is_rejects_a_replacement_space() { + let state = test_server_state().await; + let sandbox = test_sandbox("respace", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + seed_log_lines(&state, &id, 1); + let original = state.tracing_log_bus.cursor_space(&id).unwrap().epoch; + assert!(cursor_space_is(&state, &id, original)); + + // Teardown alone leaves no space to point into. + state.tracing_log_bus.remove(&id); + assert!(state.tracing_log_bus.cursor_space(&id).is_none()); + assert!(!cursor_space_is(&state, &id, original)); + + // The republish installs a replacement renumbered from 1. Its seqs + // overlap the retired space's, so only the epoch separates them. + seed_log_lines(&state, &id, 1); + let replacement = state.tracing_log_bus.cursor_space(&id).unwrap(); + assert_ne!(replacement.epoch, original); + assert_eq!(replacement.highest_seq, 1); + assert!(!cursor_space_is(&state, &id, original)); + assert!(cursor_space_is(&state, &id, replacement.epoch)); + } + + #[tokio::test] + async fn resume_from_reset_cursor_space_terminates_out_of_range() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("reset", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // The reported repro. Before cursors carried an epoch, a bare number + // was all the server had: 2 <= 3 passed the old "is this plausible?" + // bound, the tail replayed only seq 3, and the new space's seqs 1 and 2 + // -- real, unseen events -- were silently swallowed as duplicates. + seed_log_lines(&state, &id, 2); + let retired_cursor = cursor_token(&state, &id, 2); + + // Teardown retires the space; the next publish starts over at seq 1, + // indistinguishable by number from the client's view of a restart. + state.tracing_log_bus.remove(&id); + seed_log_lines(&state, &id, 3); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: retired_cursor, + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + let snap = stream.next().await.unwrap().unwrap(); + assert!(snap.cursor.is_empty()); + + let item = stream + .next() + .await + .unwrap() + .expect_err("a cursor from a retired space must terminate the stream"); + assert_eq!(item.code(), tonic::Code::OutOfRange, "{item:?}"); + assert!( + item.message().contains("empty resume_after_cursor"), + "status must tell the client to restart without a cursor, not retry: {}", + item.message() + ); + + // Nothing from the new space may be delivered before the error: a + // partial stream would read as "here is everything after your cursor". + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn resume_from_reset_cursor_space_rejected_when_new_space_is_shorter() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("reset-short", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // The shape the old numeric bound did catch (5 > 2), kept so it keeps + // passing -- but now it fails for the reason that generalizes. + seed_log_lines(&state, &id, 5); + let retired_cursor = cursor_token(&state, &id, 5); + state.tracing_log_bus.remove(&id); + seed_log_lines(&state, &id, 2); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: retired_cursor, + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + assert!(stream.next().await.unwrap().unwrap().cursor.is_empty()); + let err = stream.next().await.unwrap().expect_err("out of range"); + assert_eq!(err.code(), tonic::Code::OutOfRange, "{err:?}"); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn resume_from_reset_cursor_space_rejected_when_seq_is_within_new_range() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("reset-within", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // Seq 1 sits comfortably inside the new space's 1..=3, so every numeric + // bound accepts it. Only the epoch distinguishes the two spaces. This is + // the assertion the pre-fix design structurally could not make. + seed_log_lines(&state, &id, 3); + let retired_cursor = cursor_token(&state, &id, 1); + state.tracing_log_bus.remove(&id); + seed_log_lines(&state, &id, 3); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: retired_cursor, + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + assert!(stream.next().await.unwrap().unwrap().cursor.is_empty()); + let err = stream.next().await.unwrap().expect_err("out of range"); + assert_eq!(err.code(), tonic::Code::OutOfRange, "{err:?}"); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn resume_after_remove_without_republish_terminates_out_of_range() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("reset-empty", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // No space at all: nothing has been published since teardown. The buses + // look merely empty, so `tail_after` reports no gap -- "caught up" would + // be the wrong reading, because the client's events are gone. + seed_log_lines(&state, &id, 3); + let retired_cursor = cursor_token(&state, &id, 3); + state.tracing_log_bus.remove(&id); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: retired_cursor, + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + assert!(stream.next().await.unwrap().unwrap().cursor.is_empty()); + let err = stream.next().await.unwrap().expect_err("out of range"); + assert_eq!(err.code(), tonic::Code::OutOfRange, "{err:?}"); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn resume_with_cursor_from_another_sandbox_terminates_out_of_range() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let a = test_sandbox("epoch-a", Vec::new()); + let b = test_sandbox("epoch-b", Vec::new()); + state.store.put_message(&a).await.unwrap(); + state.store.put_message(&b).await.unwrap(); + let a_id = a.object_id().to_string(); + let b_id = b.object_id().to_string(); + + // Epochs are per sandbox, not per process. A token valid for A must not + // address B's space, even though both are on seq 1..=3 right now. + seed_log_lines(&state, &a_id, 3); + seed_log_lines(&state, &b_id, 3); + let a_cursor = cursor_token(&state, &a_id, 1); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: b.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: a_cursor, + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + assert!(stream.next().await.unwrap().unwrap().cursor.is_empty()); + let err = stream.next().await.unwrap().expect_err("out of range"); + assert_eq!(err.code(), tonic::Code::OutOfRange, "{err:?}"); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn resume_with_cursor_ahead_of_the_space_terminates_out_of_range() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("ahead", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + seed_log_lines(&state, &id, 2); + // Right epoch, but a seq this space has never issued: only a fabricated + // token gets here. Accepting it would pin the cutoff above every future + // event and stall the stream silently. + let ahead = cursor_token(&state, &id, 99); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: ahead, + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + assert!(stream.next().await.unwrap().unwrap().cursor.is_empty()); + let err = stream.next().await.unwrap().expect_err("out of range"); + assert_eq!(err.code(), tonic::Code::OutOfRange, "{err:?}"); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn resume_with_malformed_cursor_rejects_invalid_argument() { + let state = test_server_state().await; + let sandbox = test_sandbox("malformed", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + seed_log_lines(&state, &id, 3); + + // Input validation, so it fails the RPC before any stream exists rather + // than arriving as the first item of an apparently-healthy stream. + for raw in ["5", "v1:not-a-uuid:0", &foreign_cursor(1)[..40]] { + let err = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: raw.to_string(), + ..Default::default() + }), + ) + .await + .expect_err("malformed cursor must fail the call"); + assert_eq!(err.code(), tonic::Code::InvalidArgument, "{raw:?}: {err:?}"); + assert!( + !err.message().contains(raw), + "status must not echo the client token: {}", + err.message() + ); + } + } + + #[tokio::test] + async fn resume_with_foreign_epoch_terminates_out_of_range() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("foreign", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + seed_log_lines(&state, &id, 3); + + // Well-formed, so it clears input validation, but from an epoch this + // gateway never minted -- the shape a reconnect to another replica + // takes. + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + resume_after_cursor: foreign_cursor(1), + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut stream = response.into_inner(); + assert!(stream.next().await.unwrap().unwrap().cursor.is_empty()); + let err = stream.next().await.unwrap().expect_err("out of range"); + assert_eq!(err.code(), tonic::Code::OutOfRange, "{err:?}"); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn watch_delivers_each_event_once_during_init_race() { + use tokio_stream::StreamExt as _; + + let state = test_server_state().await; + let sandbox = test_sandbox("race", Vec::new()); + state.store.put_message(&sandbox).await.unwrap(); + let id = sandbox.object_id().to_string(); + + // Seed events that land in the tail before the watch subscribes. + seed_log_lines(&state, &id, 5); + + let response = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: sandbox.object_name().to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + follow_logs: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + // Publish more concurrently with producer initialization. Some of these + // can land after the broadcast subscription but before the tail read, + // putting them in both replay and the live receiver. + for i in 5..15 { + state + .tracing_log_bus + .publish_external(openshell_core::proto::SandboxLogLine { + sandbox_id: id.clone(), + event_time: openshell_core::time::timestamp_from_millis(i64::from(i)).ok(), + level: "INFO".to_string(), + target: "test".to_string(), + message: format!("line {i}"), + source: "gateway".to_string(), + ..Default::default() + }); + } + + let mut stream = response.into_inner(); + let mut cursors = Vec::new(); + while let Ok(Some(item)) = + tokio::time::timeout(std::time::Duration::from_millis(200), stream.next()).await + { + let evt = item.unwrap(); + if !evt.cursor.is_empty() { + cursors.push(seq_of(&evt)); + } + } + + // Every delivered cursor is unique (no double delivery) and monotonically + // increasing (replay ordered, then live in cursor order for one source). + let mut sorted = cursors.clone(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!( + sorted.len(), + cursors.len(), + "duplicate cursors delivered: {cursors:?}" + ); + assert_eq!( + cursors, sorted, + "cursors not delivered in order: {cursors:?}" + ); + } + #[tokio::test] async fn delete_handler_ends_telemetry_for_the_resolved_sandbox_id() { let state = test_server_state().await; diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index ee3ef1768e..c0a12facc7 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -48,6 +48,7 @@ mod tls; pub(crate) mod tls_test_utils; pub mod tracing_bus; mod tracing_setup; +mod watch_cursor; mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; diff --git a/crates/openshell-server/src/sandbox_watch.rs b/crates/openshell-server/src/sandbox_watch.rs index 3a62fce398..679912d352 100644 --- a/crates/openshell-server/src/sandbox_watch.rs +++ b/crates/openshell-server/src/sandbox_watch.rs @@ -7,8 +7,8 @@ use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; use std::time::Duration; +use openshell_core::proto::SandboxStreamWarning; use tokio::sync::{broadcast, watch}; -use tonic::Status; use crate::persistence::Store; use openshell_core::proto::Sandbox; @@ -34,6 +34,7 @@ impl SandboxWatchBus { } } + /// Private method to register sandbox in the `SandboxWatchBus` registry if it does not exist. fn sender_for(&self, sandbox_id: &str) -> broadcast::Sender<()> { let mut inner = self.inner.lock().expect("sandbox watch bus lock poisoned"); inner @@ -132,13 +133,24 @@ pub fn spawn_store_poller( }); } -/// Helper to translate broadcast lag into a gRPC status. -pub fn broadcast_to_status(err: broadcast::error::RecvError) -> Status { - match err { - broadcast::error::RecvError::Closed => Status::cancelled("stream closed"), - broadcast::error::RecvError::Lagged(n) => { - Status::resource_exhausted(format!("watch stream lagged; dropped {n} messages")) - } +/// Build the warning payload emitted when a watch broadcast receiver lags. +/// +/// Broadcast lag is recoverable: the receiver skips ahead to the oldest +/// surviving message, so the stream continues after surfacing this warning +/// instead of terminating. +pub fn lag_warning(n: u64) -> SandboxStreamWarning { + SandboxStreamWarning { + message: format!("watch stream lagged; dropped {n} messages"), + } +} + +/// Wrap [`lag_warning`] in a `SandboxStreamEvent` ready to send on the stream. +pub fn lag_warning_event(n: u64) -> openshell_core::proto::SandboxStreamEvent { + use openshell_core::proto::sandbox_stream_event::Payload; + openshell_core::proto::SandboxStreamEvent { + payload: Some(Payload::Warning(lag_warning(n))), + // Warnings are not part of the resumable log/platform sequence. + cursor: String::new(), } } @@ -189,6 +201,49 @@ mod tests { bus.remove("nonexistent"); } + #[test] + fn lag_warning_reports_dropped_count() { + let warning = lag_warning(7); + assert!( + warning.message.contains('7'), + "message: {}", + warning.message + ); + assert!( + warning.message.contains("lagged"), + "message: {}", + warning.message + ); + } + + #[test] + fn lag_warning_event_wraps_warning_payload() { + use openshell_core::proto::sandbox_stream_event::Payload; + let evt = lag_warning_event(3); + match evt.payload { + Some(Payload::Warning(w)) => assert!(w.message.contains('3')), + other => panic!("expected Warning payload, got {other:?}"), + } + } + + // Broadcast lag is recoverable at the tokio layer: after `Lagged`, the same + // receiver keeps yielding the oldest surviving messages instead of closing. + #[tokio::test] + async fn lagged_receiver_recovers_after_lag() { + const N: usize = 4; + let (tx, mut rx) = broadcast::channel(N); + for _ in 0..=N { + let _ = tx.send(()); + } + + let err = rx.recv().await.expect_err("expected Lagged"); + assert!(matches!(err, broadcast::error::RecvError::Lagged(_))); + + // The receiver is still usable: after lag it resumes at the oldest + // surviving message instead of closing. + assert!(rx.recv().await.is_ok(), "receiver should recover after lag"); + } + #[tokio::test] async fn shared_store_poller_notifies_remote_resource_version_change() { let store = Arc::new(crate::persistence::test_store().await); diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 1b40c62bb1..77fe7937a0 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -118,8 +118,15 @@ mod tests { const STORAGE_V1_SCHEMA_SHA256: &str = "d68401809d8cea445c35233ef32412bbd041cb2ac5acaf368a0d0bf74d2ddf17"; + // Carries both this branch's opaque watch cursor and the well-known time + // types from main. The cursor change adds `WatchSandboxRequest` + // `.resume_after_cursor` and `SandboxStreamEvent.cursor` as strings; both + // fields are unreleased, added on this branch, so no client depends on an + // earlier type. It adds no messages or enums and touches no stored type, + // which is why the durable and overlap fingerprints below are main's + // values unchanged. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "07e889beb43942535d80d635c490f3fb6c6f3e554f3087c101deec8c05502883"; + "fd8a5cad432441be1fe8891332b56f52335c208cc7283dcd93e385e7ae66c6da"; const DURABLE_SCHEMA_SHA256: &str = "9eeaa29dfba187bff69fb7bc4f9a13a0f1d7be3f7049a38c8f0e20ce77ec7d8b"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = diff --git a/crates/openshell-server/src/tracing_bus.rs b/crates/openshell-server/src/tracing_bus.rs index 91db86c275..fea7abad5c 100644 --- a/crates/openshell-server/src/tracing_bus.rs +++ b/crates/openshell-server/src/tracing_bus.rs @@ -4,7 +4,7 @@ //! Capture openshell-server tracing logs for streaming over gRPC. use std::collections::{HashMap, VecDeque}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard}; use openshell_core::proto::{SandboxLogLine, SandboxStreamEvent}; use openshell_ocsf::OCSF_TARGET; @@ -12,18 +12,174 @@ use tokio::sync::broadcast; use tracing::{Event, Subscriber}; use tracing_subscriber::Layer; use tracing_subscriber::layer::Context; +use uuid::Uuid; + +use crate::watch_cursor::WatchCursor; /// Bus that publishes server log lines keyed by sandbox id. #[derive(Debug, Clone)] pub struct TracingLogBus { inner: Arc>, pub(crate) platform_event_bus: PlatformEventBus, + seq: SeqAllocator, } -#[derive(Debug)] +#[derive(Debug, Clone)] struct Inner { - per_id: HashMap>, - tails: HashMap>, + per_id: HashMap, +} + +/// A buffered or broadcast stream event paired with its raw sequence number. +/// +/// The wire `SandboxStreamEvent.cursor` is an opaque token; ordering decisions +/// must not depend on its encoding. Carrying the seq alongside keeps every +/// comparison on the watch path numeric, so no consumer parses the token to +/// decide whether an event has already been delivered. +#[derive(Debug, Clone)] +pub(crate) struct CursoredEvent { + pub(crate) seq: u64, + pub(crate) event: SandboxStreamEvent, +} + +#[derive(Debug, Clone)] +struct PerSandbox { + sender: broadcast::Sender, + tail: VecDeque, + /// Highest seq this bus has evicted from `tail`. 0 = nothing trimmed. + /// + /// Under the shared cursor space each bus's tail is non-contiguous in the + /// global seq (the other bus owns the missing seqs), so a resume gap can + /// only be judged by what *this* bus actually dropped. + last_trimmed_seq: u64, +} + +impl PerSandbox { + fn new() -> Self { + let (tx, _rx) = broadcast::channel(1024); + Self { + sender: tx, + tail: VecDeque::new(), + last_trimmed_seq: 0, + } + } +} + +/// The requested resume cursor is older than the oldest buffered event; +/// the events between them were trimmed and cannot be replayed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ResumeGap { + pub requested_after: u64, + pub oldest_available: u64, +} + +/// One sandbox's cursor space: an identity plus its running sequence. +/// +/// The epoch is what makes a cursor verifiable. Sequence numbers restart at 1 +/// whenever a space is recreated, so a bare number from a previous space can +/// look like a valid position in the current one. Minting a fresh epoch with +/// the entry means every reset produces a distinguishable space, and a resume +/// cursor can be checked against the identity that issued it rather than +/// against a plausible range. +#[derive(Debug, Clone)] +struct CursorSpace { + epoch: Uuid, + next: u64, +} + +/// Identity and extent of a sandbox's current cursor space. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct CursorSpaceInfo { + pub(crate) epoch: Uuid, + /// Highest sequence issued so far. `0` means nothing published yet. + pub(crate) highest_seq: u64, +} + +/// Per-sandbox monotonic sequence allocator. +/// +/// Shared across the resumable buses (`TracingLogBus`, `PlatformEventBus`) so +/// cursors are unique and strictly ordered within a single sandbox's merged +/// stream. Stamping at publish time keeps tail cursors stable across client +/// reconnects, which is what a single `resume_after_cursor` needs. +#[derive(Debug, Clone, Default)] +struct SeqAllocator { + inner: Arc>>, +} + +impl SeqAllocator { + /// Lock the cursor space. + /// + /// Publication and teardown each hold this guard across their bus-map + /// mutation, which is what keeps cursors monotonic. Allocating and then + /// releasing would let a teardown reset the counter in between, so the + /// in-flight event lands in a freshly recreated entry carrying a cursor from + /// the old space while the next publish restarts at 1. + /// + /// The lock order is always allocator -> bus map. No path takes a bus map + /// lock and then reaches for the allocator, so the nesting cannot deadlock. + fn lock(&self) -> MutexGuard<'_, HashMap> { + self.inner.lock().expect("seq allocator lock poisoned") + } + + /// Take the next `(epoch, seq)` for this sandbox from a locked space. + /// + /// Seq starts at 1 so an empty `resume_after_cursor` means "from the + /// beginning" without skipping event 1. + /// + /// The epoch is minted on the vacant path only — once per cursor space, not + /// once per publish — so generating it under the allocator lock costs + /// nothing on the hot path. + fn next_locked(spaces: &mut HashMap, sandbox_id: &str) -> (Uuid, u64) { + let space = spaces + .entry(sandbox_id.to_string()) + .or_insert_with(|| CursorSpace { + epoch: Uuid::new_v4(), + next: 1, + }); + let seq = space.next; + space.next += 1; + (space.epoch, seq) + } + + /// Identity and extent of this sandbox's space, or `None` when no event has + /// been published into it. + /// + /// A resume cursor is validated against this: a different epoch means the + /// cursor belongs to a space that no longer exists, and `None` means there + /// is no space to resume into at all. An empty `tail_after` result is not + /// on its own proof that a cursor is still valid. + fn space(&self, sandbox_id: &str) -> Option { + self.lock().get(sandbox_id).map(|space| CursorSpaceInfo { + epoch: space.epoch, + highest_seq: space.next.saturating_sub(1), + }) + } +} + +fn tail_after_impl( + tail: &VecDeque, + last_trimmed_seq: u64, + after_seq: u64, +) -> Result, ResumeGap> { + // Gap iff this bus dropped an event the client still needs, i.e. the + // highest seq we evicted is newer than the client's position. Judged only + // on this bus's own evictions — the other bus owns the seqs missing here. + if after_seq < last_trimmed_seq { + return Err(ResumeGap { + requested_after: after_seq, + oldest_available: last_trimmed_seq + 1, + }); + } + + // Skippable events (seq <= after_seq) are the oldest, at the front, so a + // take-while would stop before reaching the wanted ones. Filter the whole + // tail instead; order is preserved and caught-up yields an empty vec. + let res: Vec = tail + .iter() + .filter(|cursored| cursored.seq > after_seq) + .cloned() + .collect(); + + Ok(res) } impl Default for TracingLogBus { @@ -35,12 +191,15 @@ impl Default for TracingLogBus { impl TracingLogBus { #[must_use] pub fn new() -> Self { + // One allocator, shared with the platform event bus so both draw from + // a single per-sandbox cursor space. + let seq = SeqAllocator::default(); Self { inner: Arc::new(Mutex::new(Inner { per_id: HashMap::new(), - tails: HashMap::new(), })), - platform_event_bus: PlatformEventBus::new(), + platform_event_bus: PlatformEventBus::new(seq.clone()), + seq, } } @@ -51,42 +210,80 @@ impl TracingLogBus { } } - fn sender_for(&self, sandbox_id: &str) -> broadcast::Sender { + fn sender_for(&self, sandbox_id: &str) -> broadcast::Sender { let mut inner = self.inner.lock().expect("tracing bus lock poisoned"); inner .per_id .entry(sandbox_id.to_string()) - .or_insert_with(|| { - let (tx, _rx) = broadcast::channel(1024); - tx - }) + .or_insert_with(PerSandbox::new) + .sender .clone() } - pub fn subscribe(&self, sandbox_id: &str) -> broadcast::Receiver { + pub(crate) fn subscribe(&self, sandbox_id: &str) -> broadcast::Receiver { self.sender_for(sandbox_id).subscribe() } - /// Remove all bus entries for the given sandbox id. + /// Remove all bus entries for the given sandbox id, including the platform + /// event bus that shares this bus's cursor allocator. + /// + /// This drops the broadcast senders (closing any active receivers with + /// `RecvError::Closed`) and frees the tail buffers. + /// + /// The whole sequence runs under the cursor-space lock, so it is atomic + /// against publication on either bus. Clearing the maps first is not enough + /// on its own: a publisher that had already allocated a cursor would insert + /// it into a recreated entry after the maps were cleared, and the next + /// publisher would restart at 1 behind it. /// - /// This drops the broadcast sender (closing any active receivers with - /// `RecvError::Closed`) and frees the tail buffer. + /// Dropping the allocator entry retires the epoch with it, so the next + /// publish mints a new one. Cursors handed out before this call are + /// therefore rejected on resume rather than mistaken for positions in the + /// replacement space. Both broadcast senders are dropped here too, so a + /// stream that was live across the reset ends rather than silently + /// continuing into a different space. pub fn remove(&self, sandbox_id: &str) { - let mut inner = self.inner.lock().expect("tracing bus lock poisoned"); - inner.per_id.remove(sandbox_id); - inner.tails.remove(sandbox_id); + let mut spaces = self.seq.lock(); + { + let mut inner = self.inner.lock().expect("tracing bus lock poisoned"); + inner.per_id.remove(sandbox_id); + } + // Takes only the platform bus map lock; never reaches for `spaces`. + self.platform_event_bus.remove(sandbox_id); + spaces.remove(sandbox_id); } - pub fn tail(&self, sandbox_id: &str, max: usize) -> Vec { + pub(crate) fn tail(&self, sandbox_id: &str, max: usize) -> Vec { let inner = self.inner.lock().expect("tracing bus lock poisoned"); inner - .tails + .per_id .get(sandbox_id) - .map(|d| d.iter().rev().take(max).cloned().collect::>()) + .map(|d| d.tail.iter().rev().take(max).cloned().collect::>()) .unwrap_or_default() .into_iter() .rev() - .collect() + .collect::>() + } + + /// Identity and extent of this sandbox's current cursor space. + /// + /// `None` means nothing has been published for the sandbox, so there is no + /// space to resume into. Takes only the allocator lock, never the bus map, + /// so it cannot invert the documented allocator -> bus map order. + pub(crate) fn cursor_space(&self, sandbox_id: &str) -> Option { + self.seq.space(sandbox_id) + } + + pub(crate) fn tail_after( + &self, + sandbox_id: &str, + after_seq: u64, + ) -> Result, ResumeGap> { + let inner = self.inner.lock().expect("tracing bus lock poisoned"); + inner.per_id.get(sandbox_id).map_or_else( + || Ok(Vec::new()), + |per| tail_after_impl(&per.tail, per.last_trimmed_seq, after_seq), + ) } /// Publish a log line from an external source (e.g., sandbox push). @@ -99,6 +296,9 @@ impl TracingLogBus { payload: Some(openshell_core::proto::sandbox_stream_event::Payload::Log( log.clone(), )), + // Placeholder: publish() stamps the real cursor from the sandbox + // cursor space. + cursor: String::new(), }; self.publish(&log.sandbox_id, evt, Self::DEFAULT_TAIL); } @@ -106,15 +306,27 @@ impl TracingLogBus { /// Default tail buffer capacity (lines per sandbox). const DEFAULT_TAIL: usize = 2000; - fn publish(&self, sandbox_id: &str, event: SandboxStreamEvent, tail_cap: usize) { - let tx = self.sender_for(sandbox_id); - let _ = tx.send(event.clone()); + fn publish(&self, sandbox_id: &str, mut event: SandboxStreamEvent, tail_cap: usize) { + // Hold the cursor space across the tail insert so a teardown cannot + // reset the counter between allocation and insertion. Lock order is + // allocator -> bus map, matching `remove`. + let mut spaces = self.seq.lock(); + let (epoch, seq) = SeqAllocator::next_locked(&mut spaces, sandbox_id); + event.cursor = WatchCursor::new(epoch, seq).encode(); let mut inner = self.inner.lock().expect("tracing bus lock poisoned"); - let deque = inner.tails.entry(sandbox_id.to_string()).or_default(); - deque.push_back(event); - while deque.len() > tail_cap { - deque.pop_front(); + let per = inner + .per_id + .entry(sandbox_id.to_string()) + .or_insert_with(PerSandbox::new); + + let cursored = CursoredEvent { seq, event }; + let _ = per.sender.send(cursored.clone()); + per.tail.push_back(cursored); + while per.tail.len() > tail_cap { + if let Some(trimmed) = per.tail.pop_front() { + per.last_trimmed_seq = trimmed.seq; + } } } } @@ -155,6 +367,9 @@ where payload: Some(openshell_core::proto::sandbox_stream_event::Payload::Log( log, )), + // Placeholder: publish() stamps the real cursor from the sandbox + // cursor space. + cursor: String::new(), }; self.bus.publish(&sandbox_id, evt, self.default_tail); } @@ -208,6 +423,163 @@ mod tests { } } + /// Fixed epoch for hand-built events, so encoded cursors are deterministic. + fn test_epoch() -> Uuid { + Uuid::parse_str("11111111-1111-4111-8111-111111111111").expect("valid uuid") + } + + /// Build a stream event carrying `seq` in its cursor for assertion. + fn stream_event(seq: u64) -> SandboxStreamEvent { + SandboxStreamEvent { + payload: Some(openshell_core::proto::sandbox_stream_event::Payload::Log( + make_log_event("sb", &seq.to_string()), + )), + cursor: WatchCursor::new(test_epoch(), seq).encode(), + } + } + + /// Build a buffered event stamped at `seq`. + fn cursored(seq: u64) -> CursoredEvent { + CursoredEvent { + seq, + event: stream_event(seq), + } + } + + /// Build a contiguous tail with seqs `lo..=hi`. + fn tail_of(lo: u64, hi: u64) -> VecDeque { + (lo..=hi).map(cursored).collect() + } + + /// Extract seqs from a run of buffered events, in order. + fn cursors(events: &[CursoredEvent]) -> Vec { + events.iter().map(|c| c.seq).collect() + } + + #[test] + fn tail_after_impl_empty_tail_returns_empty() { + let tail = VecDeque::new(); + // Nothing trimmed (last_trimmed_seq = 0): any cursor is serviceable. + assert!(tail_after_impl(&tail, 0, 0).unwrap().is_empty()); + assert!(tail_after_impl(&tail, 0, 42).unwrap().is_empty()); + } + + #[test] + fn tail_after_impl_from_zero_returns_all() { + let tail = tail_of(1, 5); + let events = tail_after_impl(&tail, 0, 0).expect("serviceable"); + assert_eq!(cursors(&events), vec![1, 2, 3, 4, 5]); + } + + #[test] + fn tail_after_impl_mid_range_returns_newer_in_order() { + let tail = tail_of(1, 5); + let events = tail_after_impl(&tail, 0, 3).expect("serviceable"); + assert_eq!(cursors(&events), vec![4, 5]); + } + + #[test] + fn tail_after_impl_caught_up_returns_empty() { + let tail = tail_of(1, 5); + // Cursor at the newest seq: nothing newer, but not a gap. + assert!(tail_after_impl(&tail, 0, 5).expect("ok").is_empty()); + } + + #[test] + fn tail_after_impl_future_cursor_returns_empty() { + let tail = tail_of(1, 5); + // Cursor beyond newest (client claims to have seen more than exists): + // still serviceable, just nothing to send. + assert!(tail_after_impl(&tail, 0, 99).expect("ok").is_empty()); + } + + #[test] + fn tail_after_impl_boundary_at_last_trimmed_is_serviceable() { + // Bus trimmed up to seq 2, retains 3..=5. Client saw exactly 2, so + // nothing they still need was dropped. + let tail = tail_of(3, 5); + let events = tail_after_impl(&tail, 2, 2).expect("serviceable"); + assert_eq!(cursors(&events), vec![3, 4, 5]); + } + + #[test] + fn tail_after_impl_gap_returns_err() { + // Bus trimmed up to seq 2, retains 3..=5. Client wants everything after + // 1, but seq 2 was evicted and cannot be replayed. + let tail = tail_of(3, 5); + let err = tail_after_impl(&tail, 2, 1).expect_err("gap"); + assert_eq!( + err, + ResumeGap { + requested_after: 1, + oldest_available: 3, + } + ); + } + + #[test] + fn tail_after_impl_non_contiguous_tail_no_false_gap() { + // Simulate the shared cursor space: this bus only owns seqs 2 and 4 + // (the other bus owns 1 and 3), and never trimmed. Resuming from 0 must + // not report a gap just because seq 1 is absent here. + let tail: VecDeque = [cursored(2), cursored(4)].into_iter().collect(); + let events = tail_after_impl(&tail, 0, 0).expect("no gap"); + assert_eq!(cursors(&events), vec![2, 4]); + } + + #[test] + fn tracing_log_bus_tail_after_serviceable_and_missing() { + let bus = TracingLogBus::new(); + let sandbox_id = "sb-ta"; + for _ in 0..3 { + bus.publish_external(make_log_event(sandbox_id, "line")); + } + // Cursors start at 1, so three publishes are seqs 1,2,3. + assert_eq!( + cursors(&bus.tail_after(sandbox_id, 0).unwrap()), + vec![1, 2, 3] + ); + assert_eq!(cursors(&bus.tail_after(sandbox_id, 2).unwrap()), vec![3]); + // Unknown sandbox: no entry, nothing buffered, no gap. + assert!(bus.tail_after("nope", 5).unwrap().is_empty()); + } + + #[test] + fn platform_event_bus_tail_after_serviceable() { + let bus = TracingLogBus::new(); + let platform = &bus.platform_event_bus; + let sandbox_id = "sb-pe"; + for _ in 0..3 { + platform.publish(sandbox_id, stream_event(0)); + } + // Shared allocator, but only the platform bus published here, so its + // seqs are 1,2,3. + assert_eq!( + cursors(&platform.tail_after(sandbox_id, 0).unwrap()), + vec![1, 2, 3] + ); + assert_eq!( + cursors(&platform.tail_after(sandbox_id, 1).unwrap()), + vec![2, 3] + ); + } + + #[test] + fn shared_allocator_interleaves_cursors_across_buses() { + let bus = TracingLogBus::new(); + let sandbox_id = "sb-mix"; + // Interleave log and platform publishes; the shared allocator gives + // each a unique, increasing cursor in one merged space. + bus.publish_external(make_log_event(sandbox_id, "a")); // seq 1 + bus.platform_event_bus.publish(sandbox_id, stream_event(0)); // seq 2 + bus.publish_external(make_log_event(sandbox_id, "b")); // seq 3 + + let logs = cursors(&bus.tail_after(sandbox_id, 0).unwrap()); + let events = cursors(&bus.platform_event_bus.tail_after(sandbox_id, 0).unwrap()); + assert_eq!(logs, vec![1, 3]); + assert_eq!(events, vec![2]); + } + #[test] fn tracing_log_bus_remove_cleans_up_all_maps() { let bus = TracingLogBus::new(); @@ -227,6 +599,132 @@ mod tests { assert!(bus.tail(sandbox_id, 10).is_empty()); } + #[test] + fn cursor_space_is_absent_until_first_publish() { + let bus = TracingLogBus::new(); + let sandbox_id = "sb-epoch-lazy"; + + assert_eq!(bus.cursor_space(sandbox_id), None); + + // Subscribing must not mint a space. `sender_for` touches only the bus + // map, never the allocator -- and the watch handler subscribes before + // it validates the resume cursor. If a subscription could manufacture + // an epoch, a client resuming against a torn-down sandbox would create + // the very space its stale cursor is then checked against. + let _log_rx = bus.subscribe(sandbox_id); + let _platform_rx = bus.platform_event_bus.subscribe(sandbox_id); + assert_eq!(bus.cursor_space(sandbox_id), None); + + bus.publish_external(make_log_event(sandbox_id, "first")); + let space = bus.cursor_space(sandbox_id).expect("space after publish"); + assert_eq!(space.highest_seq, 1); + } + + #[test] + fn publish_after_remove_starts_a_new_epoch() { + // The defect this whole mechanism exists for: teardown restarts seqs at + // 1, so a cursor from before the reset is numerically indistinguishable + // from a position in the new space. A fresh epoch makes it distinct. + let bus = TracingLogBus::new(); + let sandbox_id = "sb-epoch-reset"; + + bus.publish_external(make_log_event(sandbox_id, "a")); + bus.publish_external(make_log_event(sandbox_id, "b")); + let before = bus.cursor_space(sandbox_id).expect("space exists"); + assert_eq!(before.highest_seq, 2); + + bus.remove(sandbox_id); + assert_eq!(bus.cursor_space(sandbox_id), None); + + bus.publish_external(make_log_event(sandbox_id, "c")); + let after = bus.cursor_space(sandbox_id).expect("space recreated"); + + assert_ne!(before.epoch, after.epoch, "teardown must retire the epoch"); + // Seq alone cannot tell the spaces apart -- that is the point. + assert_eq!(after.highest_seq, 1); + } + + #[test] + fn log_and_platform_publishes_share_one_epoch() { + // Both buses draw from one allocator, so a cursor observed on either + // resumes both. Separate epochs would make a merged resume impossible. + let bus = TracingLogBus::new(); + let sandbox_id = "sb-epoch-shared"; + + bus.publish_external(make_log_event(sandbox_id, "a")); + let after_log = bus.cursor_space(sandbox_id).expect("space exists"); + + bus.platform_event_bus.publish(sandbox_id, stream_event(0)); + let after_platform = bus.cursor_space(sandbox_id).expect("space exists"); + + assert_eq!(after_log.epoch, after_platform.epoch); + assert_eq!(after_platform.highest_seq, 2); + + let log_cursor = &bus.tail(sandbox_id, 10)[0].event.cursor; + let platform_cursor = &bus.platform_event_bus.tail(sandbox_id, 10)[0].event.cursor; + assert_eq!( + WatchCursor::parse(log_cursor).expect("valid").epoch, + WatchCursor::parse(platform_cursor).expect("valid").epoch, + ); + } + + #[test] + fn concurrent_publish_and_remove_keeps_cursors_in_one_ascending_space() { + // Teardown retires the cursor space while both buses can still accept a + // publish. Unless the whole sequence is atomic against publication, a + // publisher that allocated before the reset inserts its old cursor into + // a recreated entry, and the next publisher restarts at 1 behind it -- + // leaving a tail whose cursors go backwards, or worse, one that mixes + // two epochs. + // + // Both halves matter. Resume validation trusts that every cursor in a + // tail belongs to the epoch `cursor_space` reports, so a mixed tail + // would let a cursor pass the epoch check and still address the wrong + // events. + let bus = TracingLogBus::new(); + let sandbox_id = "sb-race"; + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let barrier = Arc::new(std::sync::Barrier::new(5)); + + let publishers: Vec<_> = (0..4) + .map(|_| { + let bus = bus.clone(); + let stop = Arc::clone(&stop); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + while !stop.load(std::sync::atomic::Ordering::Relaxed) { + bus.publish_external(make_log_event(sandbox_id, "x")); + } + }) + }) + .collect(); + + barrier.wait(); + // Interleave teardown with in-flight publication. Each observation is a + // sample of the tail mid-race; a non-monotonic one means an event was + // stamped from a cursor space that no longer existed when it landed. + for _ in 0..20_000 { + bus.remove(sandbox_id); + let observed: Vec = bus + .tail(sandbox_id, usize::MAX) + .iter() + .map(|c| WatchCursor::parse(&c.event.cursor).expect("bus stamps valid cursors")) + .collect(); + assert!( + observed + .windows(2) + .all(|w| w[0].epoch == w[1].epoch && w[0].seq < w[1].seq), + "tail must stay in one epoch with strictly ascending seqs, got {observed:?}" + ); + } + + stop.store(true, std::sync::atomic::Ordering::Relaxed); + for publisher in publishers { + publisher.join().expect("publisher thread panicked"); + } + } + #[test] fn tracing_log_bus_subscribe_after_remove_creates_fresh_channel() { let bus = TracingLogBus::new(); @@ -243,7 +741,7 @@ mod tests { // New publish should reach the new subscriber bus.publish_external(make_log_event(sandbox_id, "new message")); let evt = rx.try_recv().expect("should receive new event"); - assert!(evt.payload.is_some()); + assert!(evt.event.payload.is_some()); } #[test] @@ -278,13 +776,16 @@ mod tests { #[test] fn platform_event_bus_remove_cleans_up() { - let bus = PlatformEventBus::new(); + let bus = PlatformEventBus::new(SeqAllocator::default()); let sandbox_id = "sb-4"; let mut rx = bus.subscribe(sandbox_id); // Publish an event - let evt = SandboxStreamEvent { payload: None }; + let evt = SandboxStreamEvent { + payload: None, + cursor: String::new(), + }; bus.publish(sandbox_id, evt); assert!(rx.try_recv().is_ok()); @@ -300,7 +801,7 @@ mod tests { #[test] fn platform_event_bus_subscribe_after_remove_creates_fresh_channel() { - let bus = PlatformEventBus::new(); + let bus = PlatformEventBus::new(SeqAllocator::default()); let sandbox_id = "sb-5"; let _old_rx = bus.subscribe(sandbox_id); @@ -308,14 +809,17 @@ mod tests { // New subscription should work let mut new_rx = bus.subscribe(sandbox_id); - let evt = SandboxStreamEvent { payload: None }; + let evt = SandboxStreamEvent { + payload: None, + cursor: String::new(), + }; bus.publish(sandbox_id, evt); assert!(new_rx.try_recv().is_ok()); } #[test] fn platform_event_bus_remove_nonexistent_is_noop() { - let bus = PlatformEventBus::new(); + let bus = PlatformEventBus::new(SeqAllocator::default()); // Should not panic bus.remove("nonexistent"); } @@ -324,7 +828,7 @@ mod tests { fn platform_event_bus_tail_returns_buffered_events() { use openshell_core::proto::{PlatformEvent, sandbox_stream_event}; - let bus = PlatformEventBus::new(); + let bus = PlatformEventBus::new(SeqAllocator::default()); let sandbox_id = "sb-6"; // Publish some events @@ -338,6 +842,7 @@ mod tests { message: format!("Message {i}"), metadata: HashMap::new(), })), + cursor: String::new(), }; bus.publish(sandbox_id, evt); } @@ -347,8 +852,8 @@ mod tests { assert_eq!(events.len(), 5); // Verify order (oldest first) - for (i, evt) in events.iter().enumerate() { - if let Some(sandbox_stream_event::Payload::Event(ref e)) = evt.payload { + for (i, cursored) in events.iter().enumerate() { + if let Some(sandbox_stream_event::Payload::Event(ref e)) = cursored.event.payload { assert_eq!(e.reason, format!("Event{i}")); } else { panic!("expected Event payload"); @@ -358,27 +863,30 @@ mod tests { // Tail with smaller max should return most recent events let events = bus.tail(sandbox_id, 2); assert_eq!(events.len(), 2); - if let Some(sandbox_stream_event::Payload::Event(ref e)) = events[0].payload { + if let Some(sandbox_stream_event::Payload::Event(ref e)) = events[0].event.payload { assert_eq!(e.reason, "Event3"); } - if let Some(sandbox_stream_event::Payload::Event(ref e)) = events[1].payload { + if let Some(sandbox_stream_event::Payload::Event(ref e)) = events[1].event.payload { assert_eq!(e.reason, "Event4"); } } #[test] fn platform_event_bus_tail_empty_sandbox() { - let bus = PlatformEventBus::new(); + let bus = PlatformEventBus::new(SeqAllocator::default()); let events = bus.tail("nonexistent", 10); assert!(events.is_empty()); } #[test] fn platform_event_bus_remove_clears_tail() { - let bus = PlatformEventBus::new(); + let bus = PlatformEventBus::new(SeqAllocator::default()); let sandbox_id = "sb-7"; - let evt = SandboxStreamEvent { payload: None }; + let evt = SandboxStreamEvent { + payload: None, + cursor: String::new(), + }; bus.publish(sandbox_id, evt); assert_eq!(bus.tail(sandbox_id, 10).len(), 1); @@ -392,13 +900,8 @@ mod tests { /// This keeps platform events isolated from tracing capture. #[derive(Debug, Clone)] pub(crate) struct PlatformEventBus { - inner: Arc>, -} - -#[derive(Debug)] -struct PlatformEventBusInner { - senders: HashMap>, - tails: HashMap>, + inner: Arc>, + seq: SeqAllocator, } impl PlatformEventBus { @@ -406,63 +909,86 @@ impl PlatformEventBus { /// Platform events are infrequent (typically 5-10 per sandbox lifecycle). const DEFAULT_TAIL: usize = 50; - fn new() -> Self { + /// Build a platform event bus sharing `seq` with its owning `TracingLogBus` + /// so both stamp cursors from the same per-sandbox sequence. + fn new(seq: SeqAllocator) -> Self { Self { - inner: Arc::new(Mutex::new(PlatformEventBusInner { - senders: HashMap::new(), - tails: HashMap::new(), + inner: Arc::new(Mutex::new(Inner { + per_id: HashMap::new(), })), + seq, } } - fn sender_for(&self, sandbox_id: &str) -> broadcast::Sender { + fn sender_for(&self, sandbox_id: &str) -> broadcast::Sender { let mut inner = self.inner.lock().expect("platform event bus lock poisoned"); inner - .senders + .per_id .entry(sandbox_id.to_string()) - .or_insert_with(|| { - let (tx, _rx) = broadcast::channel(1024); - tx - }) + .or_insert_with(PerSandbox::new) + .sender .clone() } - pub(crate) fn subscribe(&self, sandbox_id: &str) -> broadcast::Receiver { + pub(crate) fn subscribe(&self, sandbox_id: &str) -> broadcast::Receiver { self.sender_for(sandbox_id).subscribe() } - pub(crate) fn publish(&self, sandbox_id: &str, event: SandboxStreamEvent) { - let tx = self.sender_for(sandbox_id); - let _ = tx.send(event.clone()); + pub(crate) fn publish(&self, sandbox_id: &str, mut event: SandboxStreamEvent) { + // Hold the cursor space across the tail insert (same allocator -> map + // lock order as `TracingLogBus::publish`), so teardown cannot reset the + // counter underneath an in-flight publish. + let mut spaces = self.seq.lock(); + let (epoch, seq) = SeqAllocator::next_locked(&mut spaces, sandbox_id); + event.cursor = WatchCursor::new(epoch, seq).encode(); let mut inner = self.inner.lock().expect("platform event bus lock poisoned"); - let deque = inner.tails.entry(sandbox_id.to_string()).or_default(); - deque.push_back(event); - while deque.len() > Self::DEFAULT_TAIL { - deque.pop_front(); + let per = inner + .per_id + .entry(sandbox_id.to_string()) + .or_insert_with(PerSandbox::new); + + let cursored = CursoredEvent { seq, event }; + let _ = per.sender.send(cursored.clone()); + per.tail.push_back(cursored); + while per.tail.len() > Self::DEFAULT_TAIL { + if let Some(trimmed) = per.tail.pop_front() { + per.last_trimmed_seq = trimmed.seq; + } } } /// Return buffered platform events for replay to late subscribers. - pub(crate) fn tail(&self, sandbox_id: &str, max: usize) -> Vec { + pub(crate) fn tail(&self, sandbox_id: &str, max: usize) -> Vec { let inner = self.inner.lock().expect("platform event bus lock poisoned"); inner - .tails + .per_id .get(sandbox_id) - .map(|d| d.iter().rev().take(max).cloned().collect::>()) + .map(|d| d.tail.iter().rev().take(max).cloned().collect::>()) .unwrap_or_default() .into_iter() .rev() .collect() } + pub(crate) fn tail_after( + &self, + sandbox_id: &str, + after_seq: u64, + ) -> Result, ResumeGap> { + let inner = self.inner.lock().expect("platform event bus lock poisoned"); + inner.per_id.get(sandbox_id).map_or_else( + || Ok(Vec::new()), + |per| tail_after_impl(&per.tail, per.last_trimmed_seq, after_seq), + ) + } + /// Remove the bus entry for the given sandbox id. /// /// This drops the broadcast sender, closing any active receivers, /// and frees the tail buffer. pub(crate) fn remove(&self, sandbox_id: &str) { let mut inner = self.inner.lock().expect("platform event bus lock poisoned"); - inner.senders.remove(sandbox_id); - inner.tails.remove(sandbox_id); + inner.per_id.remove(sandbox_id); } } diff --git a/crates/openshell-server/src/watch_cursor.rs b/crates/openshell-server/src/watch_cursor.rs new file mode 100644 index 0000000000..894b52aa18 --- /dev/null +++ b/crates/openshell-server/src/watch_cursor.rs @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Opaque, epoch-bound cursors for the `WatchSandbox` stream. +//! +//! A cursor is only meaningful inside the cursor space that issued it. A +//! gateway restart, or teardown of a sandbox's buses, starts a new space whose +//! sequence numbers restart at 1, so a bare number cannot tell "caught up" +//! apart from "belongs to a space that no longer exists". Binding the sequence +//! to the space's epoch makes that distinction explicit: a cursor from a dead +//! space is rejected instead of silently suppressing live events beneath it. +//! +//! # Wire contract +//! +//! `v1::<20-digit zero-padded seq>` +//! +//! The token is **opaque to clients**. The only operation a client may perform +//! is comparing two cursors from the same stream and keeping the greater one as +//! its resume point. That comparison is byte-wise: the version prefix and uuid +//! are fixed width, and the sequence is zero-padded, so lexicographic order +//! equals sequence order within one epoch. Every stream observes exactly one +//! epoch (a reset closes the stream), so the comparison is always well defined +//! where clients are allowed to use it. +//! +//! The server does **not** rely on that property. Ordering decisions on the +//! watch path run on the raw `u64` sequence carried alongside each event in +//! [`crate::tracing_bus::CursoredEvent`], so no server-side correctness +//! decision depends on the encoding. + +use std::fmt; + +use uuid::Uuid; + +/// Current cursor encoding version. +const VERSION: &str = "v1"; + +/// Zero-padded width of the sequence segment. `u64::MAX` is 20 digits. +const SEQ_WIDTH: usize = 20; + +/// Exact encoded length: `"v1"` + `':'` + hyphenated uuid + `':'` + seq. +const ENCODED_LEN: usize = VERSION.len() + 1 + 36 + 1 + SEQ_WIDTH; + +/// A resume cursor: a sequence number bound to the cursor space that issued it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WatchCursor { + pub(crate) epoch: Uuid, + pub(crate) seq: u64, +} + +/// The client supplied a cursor this server could not have issued. +/// +/// Deliberately carries no detail from the input: the token is echoed back to +/// nobody, and a single opaque reason keeps malformed input from becoming a +/// reflection vector. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CursorParseError; + +impl fmt::Display for CursorParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("resume_after_cursor is not a valid watch cursor") + } +} + +impl std::error::Error for CursorParseError {} + +impl WatchCursor { + pub(crate) const fn new(epoch: Uuid, seq: u64) -> Self { + Self { epoch, seq } + } + + /// Encode as the opaque wire token. + pub(crate) fn encode(&self) -> String { + format!( + "{VERSION}:{}:{:0SEQ_WIDTH$}", + self.epoch.as_hyphenated(), + self.seq + ) + } + + /// Parse a client-supplied token. + /// + /// Strict by design. Anything this server would not have produced is + /// rejected, so a client cannot hand back a hand-built or truncated cursor + /// and have it silently treated as a position in the current space. + pub(crate) fn parse(raw: &str) -> Result { + // Fixed-width encoding, so one length check also bounds the work done + // on hostile input. + if raw.len() != ENCODED_LEN { + return Err(CursorParseError); + } + + let mut parts = raw.split(':'); + let (Some(version), Some(epoch), Some(seq), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return Err(CursorParseError); + }; + + if version != VERSION { + return Err(CursorParseError); + } + + // Re-encode and compare so only the canonical lowercase hyphenated form + // is accepted. `Uuid::parse_str` also takes braced and simple forms, + // which would give one epoch several spellings and break the + // lexicographic ordering clients rely on. + let parsed_epoch = Uuid::parse_str(epoch).map_err(|_| CursorParseError)?; + if parsed_epoch.as_hyphenated().to_string() != epoch { + return Err(CursorParseError); + } + + if seq.len() != SEQ_WIDTH || !seq.bytes().all(|b| b.is_ascii_digit()) { + return Err(CursorParseError); + } + // 20 digits can exceed u64::MAX, so this also rejects overflow. + let seq: u64 = seq.parse().map_err(|_| CursorParseError)?; + + // Sequences start at 1; an empty cursor is the only "from the + // beginning" signal. + if seq == 0 { + return Err(CursorParseError); + } + + Ok(Self { + epoch: parsed_epoch, + seq, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn epoch() -> Uuid { + Uuid::parse_str("3f2a9c14-7b6e-4d81-9a02-1c5d8e4f7b30").expect("valid uuid") + } + + #[test] + fn encode_parse_roundtrip() { + for seq in [1, 2, 9, 10, 999, u64::MAX] { + let cursor = WatchCursor::new(epoch(), seq); + let encoded = cursor.encode(); + assert_eq!(encoded.len(), ENCODED_LEN); + assert_eq!(WatchCursor::parse(&encoded).expect("roundtrip"), cursor); + } + } + + #[test] + fn encoding_is_lexicographically_ordered_by_seq() { + // Clients are told they may compare two cursors from one stream + // byte-wise and keep the greater. That only holds because the sequence + // is zero-padded to a fixed width; dropping the padding would make + // "...:9" sort above "...:10" and silently rewind every reconnect. + let encoded: Vec = [1_u64, 2, 9, 10, 99, 100, u64::MAX] + .into_iter() + .map(|seq| WatchCursor::new(epoch(), seq).encode()) + .collect(); + + let mut sorted = encoded.clone(); + sorted.sort(); + assert_eq!(sorted, encoded, "lexicographic order must match seq order"); + } + + #[test] + fn parse_rejects_malformed() { + let e = epoch().as_hyphenated().to_string(); + let cases = [ + ("empty", String::new()), + ("bare number", "5".to_string()), + ("unpadded seq", format!("v1:{e}:5")), + ("wrong version", format!("v2:{e}:{:020}", 1)), + ( + "not a uuid", + format!("v1:not-a-uuid-not-a-uuid-not-a-uuid-x:{:020}", 1), + ), + ( + "uppercase uuid", + format!("v1:{}:{:020}", e.to_uppercase(), 1), + ), + ( + "simple uuid", + format!("v1:{}:{:020}", epoch().as_simple(), 1), + ), + ("four segments", format!("v1:{e}:{:020}:x", 1)), + ("zero seq", format!("v1:{e}:{:020}", 0)), + ("non-digit seq", format!("v1:{e}:0000000000000000000x")), + ("seq overflows u64", format!("v1:{e}:99999999999999999999")), + ("over-long input", "x".repeat(200)), + ]; + + for (name, raw) in cases { + assert_eq!( + WatchCursor::parse(&raw), + Err(CursorParseError), + "expected rejection for {name}" + ); + } + } + + #[test] + fn parse_error_does_not_echo_input() { + let rendered = CursorParseError.to_string(); + assert!(!rendered.contains("v1:"), "error must not echo the token"); + } +} diff --git a/docs/observability/accessing-logs.mdx b/docs/observability/accessing-logs.mdx index 4b755f74cc..e5981c9583 100644 --- a/docs/observability/accessing-logs.mdx +++ b/docs/observability/accessing-logs.mdx @@ -39,6 +39,25 @@ The sandbox pushes logs to the gateway over gRPC in real time. The gateway store For durable log storage, use the log files inside the sandbox or enable [OCSF JSON export](/observability/ocsf-json-export) and ship the JSONL files to an external log aggregator. +## Loss Awareness and Resume + +The watch stream behind `openshell logs` is loss-aware. Each resumable event (log line or platform event) carries an opaque `cursor` token. Status snapshots and warnings carry an empty cursor. + +Do not parse a cursor. The only supported operation is comparing two cursors observed on the same stream and keeping the greater one as the resume point. That comparison is a plain string comparison in any language. + +The gateway distinguishes recoverable from unrecoverable loss: + +- **Recoverable lag.** When a consumer falls behind and the gateway skips ahead in its buffer, the stream emits a warning event and keeps running. The warning is the signal that events were skipped. +- **Unrecoverable gap.** When a client reconnects and asks to resume after a cursor the gateway has already trimmed from its buffer, the stream ends with an `OUT_OF_RANGE` status. The client should restart observation and, if it needs the missing lines, read them from the log files inside the sandbox. + +A cursor is bound to the cursor space that issued it. A gateway restart, teardown of the sandbox's buffers, or a reconnect that lands on a different gateway replica starts a new space, and cursors from the previous one no longer refer to anything. The gateway rejects them with `OUT_OF_RANGE` instead of treating them as caught up — which would silently suppress the new space's events. A malformed cursor is rejected with `INVALID_ARGUMENT`. + +`OUT_OF_RANGE` is terminal for that cursor. Restart the watch with an empty resume cursor; retrying the same one fails identically. + +On reconnect, a client passes the highest cursor it processed as the resume point. The gateway replays only events after that cursor — logs and platform events merged in order — then resumes live delivery. The handoff from replay to live delivery is exact: an event buffered while the stream was reopening is delivered once, never twice. It is not a guarantee that nothing was lost — a warning event or an `OUT_OF_RANGE` status still reports loss, both before and after a reconnect. + +The gateway merges the log and platform event sources before emitting, so events normally arrive in ascending cursor order. That applies to the buffered tail you receive when the stream opens, to a replay after a resume, and to live delivery. The gateway does not delay an event to wait for a lower cursor that has not been published yet, so a cursor can still arrive late under concurrent publication. Track the highest cursor seen as the resume point rather than the last one received. + ## Direct Filesystem Access Start an independent shell with `sandbox exec` to read log files directly: diff --git a/proto/openshell.proto b/proto/openshell.proto index 0623943f87..dd05f0fd2b 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -2010,6 +2010,25 @@ message WatchSandboxRequest { // Minimum log level to include (e.g. "INFO", "WARN", "ERROR"). Empty means all levels. string log_min_level = 10; + + // Resume streaming after this cursor. Empty means no cursor resume: the + // server falls back to tail-limited replay controlled by log_tail_lines and + // event_tail. Otherwise set it to the highest `SandboxStreamEvent.cursor` + // already processed; the server replays only log and platform events after + // it, merged in cursor order, before resuming live delivery. If the requested + // cursor has already been trimmed from the server's buffer, the resume is + // unrecoverable and the stream terminates with OUT_OF_RANGE (see + // SandboxStreamWarning for the recoverable case). + // + // A cursor is bound to the cursor space that issued it. A gateway restart, + // teardown of the sandbox's buffers, or a reconnect to a different gateway + // replica starts a new space, and cursors from the previous one are rejected + // with OUT_OF_RANGE rather than silently suppressing live events beneath + // them. OUT_OF_RANGE is terminal for that cursor: restart the watch with an + // empty resume_after_cursor, because retrying the same token fails + // identically. A cursor this server could not have issued is rejected with + // INVALID_ARGUMENT. + string resume_after_cursor = 12; } // One event in a sandbox watch stream. @@ -2021,11 +2040,25 @@ message SandboxStreamEvent { SandboxLogLine log = 2; // One platform event. PlatformEvent event = 3; - // Warning from the server (e.g. missed messages due to lag). + // Recoverable warning from the server, e.g. messages dropped because a + // broadcast receiver lagged. The stream continues after this warning; the + // client can detect the gap from the warning itself. SandboxStreamWarning warning = 4; // Draft policy update notification. DraftPolicyUpdate draft_policy_update = 5; } + // Opaque position in this sandbox's cursor space, shared across the resumable + // log and platform event sources. Empty for non-resumable events (status + // snapshots, warnings). + // + // Do not parse this token; its encoding is not part of the contract. The only + // supported operation is comparing two non-empty cursors observed on the same + // stream and keeping the greater one, then passing it as + // WatchSandboxRequest.resume_after_cursor to resume without loss or + // duplication. That comparison is a plain byte-wise string comparison. It is + // well defined only within one stream: a stream never spans two cursor + // spaces, because a reset ends it. + string cursor = 6; } // Log line correlated to a sandbox. @@ -2044,6 +2077,11 @@ message SandboxLogLine { map fields = 7; } +// Recoverable loss notification on a watch stream. Emitted when the server +// skips ahead after a broadcast lag instead of terminating; the stream keeps +// running. Cursors are opaque, so this message is the only signal that events +// were skipped. Unrecoverable loss (a trimmed or foreign resume cursor) is +// reported as an OUT_OF_RANGE stream status, not this message. message SandboxStreamWarning { string message = 1; } diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 8b3438eff2..89e10bd26c 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -7467,9 +7467,27 @@ type WatchSandboxRequest struct { // Filter by log source (e.g. "gateway", "sandbox"). Empty means all sources. LogSources []string `protobuf:"bytes,9,rep,name=log_sources,json=logSources,proto3" json:"log_sources,omitempty"` // Minimum log level to include (e.g. "INFO", "WARN", "ERROR"). Empty means all levels. - LogMinLevel string `protobuf:"bytes,10,opt,name=log_min_level,json=logMinLevel,proto3" json:"log_min_level,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + LogMinLevel string `protobuf:"bytes,10,opt,name=log_min_level,json=logMinLevel,proto3" json:"log_min_level,omitempty"` + // Resume streaming after this cursor. Empty means no cursor resume: the + // server falls back to tail-limited replay controlled by log_tail_lines and + // event_tail. Otherwise set it to the highest `SandboxStreamEvent.cursor` + // already processed; the server replays only log and platform events after + // it, merged in cursor order, before resuming live delivery. If the requested + // cursor has already been trimmed from the server's buffer, the resume is + // unrecoverable and the stream terminates with OUT_OF_RANGE (see + // SandboxStreamWarning for the recoverable case). + // + // A cursor is bound to the cursor space that issued it. A gateway restart, + // teardown of the sandbox's buffers, or a reconnect to a different gateway + // replica starts a new space, and cursors from the previous one are rejected + // with OUT_OF_RANGE rather than silently suppressing live events beneath + // them. OUT_OF_RANGE is terminal for that cursor: restart the watch with an + // empty resume_after_cursor, because retrying the same token fails + // identically. A cursor this server could not have issued is rejected with + // INVALID_ARGUMENT. + ResumeAfterCursor string `protobuf:"bytes,12,opt,name=resume_after_cursor,json=resumeAfterCursor,proto3" json:"resume_after_cursor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WatchSandboxRequest) Reset() { @@ -7579,6 +7597,13 @@ func (x *WatchSandboxRequest) GetLogMinLevel() string { return "" } +func (x *WatchSandboxRequest) GetResumeAfterCursor() string { + if x != nil { + return x.ResumeAfterCursor + } + return "" +} + // One event in a sandbox watch stream. type SandboxStreamEvent struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -7589,7 +7614,19 @@ type SandboxStreamEvent struct { // *SandboxStreamEvent_Event // *SandboxStreamEvent_Warning // *SandboxStreamEvent_DraftPolicyUpdate - Payload isSandboxStreamEvent_Payload `protobuf_oneof:"payload"` + Payload isSandboxStreamEvent_Payload `protobuf_oneof:"payload"` + // Opaque position in this sandbox's cursor space, shared across the resumable + // log and platform event sources. Empty for non-resumable events (status + // snapshots, warnings). + // + // Do not parse this token; its encoding is not part of the contract. The only + // supported operation is comparing two non-empty cursors observed on the same + // stream and keeping the greater one, then passing it as + // WatchSandboxRequest.resume_after_cursor to resume without loss or + // duplication. That comparison is a plain byte-wise string comparison. It is + // well defined only within one stream: a stream never spans two cursor + // spaces, because a reset ends it. + Cursor string `protobuf:"bytes,6,opt,name=cursor,proto3" json:"cursor,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -7676,6 +7713,13 @@ func (x *SandboxStreamEvent) GetDraftPolicyUpdate() *DraftPolicyUpdate { return nil } +func (x *SandboxStreamEvent) GetCursor() string { + if x != nil { + return x.Cursor + } + return "" +} + type isSandboxStreamEvent_Payload interface { isSandboxStreamEvent_Payload() } @@ -7696,7 +7740,9 @@ type SandboxStreamEvent_Event struct { } type SandboxStreamEvent_Warning struct { - // Warning from the server (e.g. missed messages due to lag). + // Recoverable warning from the server, e.g. messages dropped because a + // broadcast receiver lagged. The stream continues after this warning; the + // client can detect the gap from the warning itself. Warning *SandboxStreamWarning `protobuf:"bytes,4,opt,name=warning,proto3,oneof"` } @@ -7811,6 +7857,11 @@ func (x *SandboxLogLine) GetFields() map[string]string { return nil } +// Recoverable loss notification on a watch stream. Emitted when the server +// skips ahead after a broadcast lag instead of terminating; the stream keeps +// running. Cursors are opaque, so this message is the only signal that events +// were skipped. Unrecoverable loss (a trimmed or foreign resume cursor) is +// reported as an OUT_OF_RANGE stream status, not this message. type SandboxStreamWarning struct { state protoimpl.MessageState `protogen:"open.v1"` Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` @@ -17976,7 +18027,7 @@ const file_openshell_proto_rawDesc = "" + "sandbox_id\x18\x02 \x01(\tR\tsandboxId\x12\x1a\n" + "\x05token\x18\x03 \x01(\tB\x04\x88\xb5\x18\x01R\x05token\x12C\n" + "\x0fexpiration_time\x18h \x01(\v2\x1a.google.protobuf.TimestampR\x0eexpirationTime\x12\x18\n" + - "\arevoked\x18\x05 \x01(\bR\arevokedJ\x04\b\x04\x10\x05R\rexpires_at_ms\"\xf1\x03\n" + + "\arevoked\x18\x05 \x01(\bR\arevokedJ\x04\b\x04\x10\x05R\rexpires_at_ms\"\xa1\x04\n" + "\x13WatchSandboxRequest\x12R\n" + "\x0fworkspace_scope\x18\v \x01(\v2).openshell.datamodel.v1.WorkspaceSelectorR\x0eworkspaceScope\x12\x18\n" + "\asandbox\x18\x01 \x01(\tR\asandbox\x12#\n" + @@ -17993,13 +18044,15 @@ const file_openshell_proto_rawDesc = "" + "\vlog_sources\x18\t \x03(\tR\n" + "logSources\x12\"\n" + "\rlog_min_level\x18\n" + - " \x01(\tR\vlogMinLevelJ\x04\b\b\x10\tR\flog_since_ms\"\xcc\x02\n" + + " \x01(\tR\vlogMinLevel\x12.\n" + + "\x13resume_after_cursor\x18\f \x01(\tR\x11resumeAfterCursorJ\x04\b\b\x10\tR\flog_since_ms\"\xe4\x02\n" + "\x12SandboxStreamEvent\x121\n" + "\asandbox\x18\x01 \x01(\v2\x15.openshell.v1.SandboxH\x00R\asandbox\x120\n" + "\x03log\x18\x02 \x01(\v2\x1c.openshell.v1.SandboxLogLineH\x00R\x03log\x123\n" + "\x05event\x18\x03 \x01(\v2\x1b.openshell.v1.PlatformEventH\x00R\x05event\x12>\n" + "\awarning\x18\x04 \x01(\v2\".openshell.v1.SandboxStreamWarningH\x00R\awarning\x12Q\n" + - "\x13draft_policy_update\x18\x05 \x01(\v2\x1f.openshell.v1.DraftPolicyUpdateH\x00R\x11draftPolicyUpdateB\t\n" + + "\x13draft_policy_update\x18\x05 \x01(\v2\x1f.openshell.v1.DraftPolicyUpdateH\x00R\x11draftPolicyUpdate\x12\x16\n" + + "\x06cursor\x18\x06 \x01(\tR\x06cursorB\t\n" + "\apayload\"\xdb\x02\n" + "\x0eSandboxLogLine\x12\x1d\n" + "\n" +