From 5a81d2b37b8c4499fafdfc36b2ea0010f7ccf6fc Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 22 Sep 2026 20:03:52 +0000 Subject: [PATCH 1/9] fix(network): bound chunked relay memory (#3537) Signed-off-by: Piotr Mlocek --- architecture/sandbox-limits.md | 1 + .../src/l7/rest.rs | 415 ++++++++++-------- 2 files changed, 238 insertions(+), 178 deletions(-) diff --git a/architecture/sandbox-limits.md b/architecture/sandbox-limits.md index 9045ddc359..b8d03db39d 100644 --- a/architecture/sandbox-limits.md +++ b/architecture/sandbox-limits.md @@ -95,6 +95,7 @@ streaming HTTP middleware to use the same process-wide budget. |---|---:|---| | Initial CONNECT request headers | 8 KiB | Reject the proxy request. | | Inspected HTTP/1 request headers | 16 KiB | Reject the request. | +| Streamed HTTP/1 chunk framing | 16 KiB per chunk-size line; 16 KiB and 128 fields for the complete trailer block | End the relay. Chunk payloads pass through a fixed 8 KiB buffer and do not accumulate to the declared chunk size. | | Credential-rewritten HTTP body | 256 KiB | Reject when rewriting requires a larger buffered body. | | SigV4 body signing | 10 MiB | Reject when signing requires a larger buffered body. | | GraphQL request body | 64 KiB default | Policy can set a positive `graphql_max_body_bytes`; there is no shared platform ceiling yet. | diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index f8b1f3166f..399db42301 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -67,6 +67,8 @@ async fn max_middleware_body_bytes() -> usize { chain[0].max_payload_bytes() } const RELAY_BUF_SIZE: usize = 8192; +const MAX_CHUNK_LINE_BYTES: usize = MAX_HEADER_BYTES; +const MAX_CHUNK_TRAILER_FIELDS: usize = 128; const RESPONSE_UNIT_COALESCE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(2); const HTTP_METHOD_PREFIXES: &[&[u8]] = &[ b"GET ", @@ -3113,8 +3115,8 @@ where /// Handles chunk extensions and trailers per RFC 7230. /// /// `already_forwarded` are overflow bytes that were already written to the -/// writer during header parsing. They are seeded into the parser buffer so -/// termination can still be detected when boundaries span reads. +/// writer during header parsing. The parser consumes them before reading more +/// bytes so it can detect boundaries without forwarding them twice. async fn relay_chunked( reader: &mut R, writer: &mut W, @@ -3126,43 +3128,19 @@ where W: AsyncWrite + Unpin, { let started_at = std::time::Instant::now(); - let mut read_buf = [0u8; RELAY_BUF_SIZE]; - let mut parse_buf = Vec::from(already_forwarded); - let mut forwarded_len = already_forwarded.len(); - let mut pos = 0usize; + let mut input = ChunkedRelayInput::new(already_forwarded); let mut chunk_count = 0usize; let mut chunk_payload_bytes = 0usize; - // Parse chunk-size lines + chunk payloads until final 0-size chunk, then - // parse trailers until the terminating empty trailer line. loop { - // Parse one chunk size line: "[;extensions]\r\n" - let size_line_end = loop { - if let Some(end) = find_crlf(&parse_buf, pos) { - break end; - } - if parse_buf.len().saturating_sub(pos) >= MAX_HEADER_BYTES { - return Err(miette!("Chunk-size line exceeds limit")); - } - let target_len = parse_buf - .len() - .checked_add(1) - .ok_or_else(|| miette!("Chunked body size overflow"))?; - relay_chunked_until_len( + let size_line = input + .read_line( reader, - &mut read_buf, - &mut parse_buf, - target_len, - generation_guard, "Chunked body ended before chunk-size line", + "Chunk-size line exceeds limit", ) .await?; - }; - if size_line_end.saturating_add(2).saturating_sub(pos) > MAX_HEADER_BYTES { - return Err(miette!("Chunk-size line exceeds limit")); - } - - let size_line = std::str::from_utf8(&parse_buf[pos..size_line_end]) + let size_line = std::str::from_utf8(&size_line) .into_diagnostic() .map_err(|_| miette!("Invalid UTF-8 in chunk-size line"))?; let size_token = size_line @@ -3173,59 +3151,26 @@ where let chunk_size = usize::from_str_radix(size_token, 16) .into_diagnostic() .map_err(|_| miette!("Invalid chunk size token: {size_token:?}"))?; - pos = size_line_end + 2; - flush_validated_chunked_bytes( - writer, - &parse_buf, - &mut forwarded_len, - pos, - false, - generation_guard, - ) - .await?; if chunk_size == 0 { - // Parse trailers (if any). Terminates on empty trailer line. let mut trailer_count = 0usize; + let mut trailer_bytes = 0usize; loop { - let trailer_end = loop { - if let Some(end) = find_crlf(&parse_buf, pos) { - break end; - } - if parse_buf.len().saturating_sub(pos) >= MAX_HEADER_BYTES { - return Err(miette!("Chunk trailer line exceeds limit")); - } - let target_len = parse_buf - .len() - .checked_add(1) - .ok_or_else(|| miette!("Chunked trailer size overflow"))?; - relay_chunked_until_len( + let trailer_line = input + .read_line( reader, - &mut read_buf, - &mut parse_buf, - target_len, - generation_guard, "Chunked body ended before trailer terminator", + "Chunk trailer line exceeds limit", ) .await?; - }; - if trailer_end.saturating_add(2).saturating_sub(pos) > MAX_HEADER_BYTES { - return Err(miette!("Chunk trailer line exceeds limit")); + trailer_bytes = trailer_bytes + .checked_add(trailer_line.len() + 2) + .ok_or_else(|| miette!("Chunk trailer size overflow"))?; + if trailer_bytes > MAX_HEADER_BYTES { + return Err(miette!("Chunk trailers exceed {MAX_HEADER_BYTES} bytes")); } - - let trailer_line = &parse_buf[pos..trailer_end]; - let trailer_is_empty = trailer_line.is_empty(); - pos = trailer_end + 2; - flush_validated_chunked_bytes( - writer, - &parse_buf, - &mut forwarded_len, - pos, - trailer_is_empty, - generation_guard, - ) - .await?; - if trailer_is_empty { + if trailer_line.is_empty() { + input.flush_pending(writer, generation_guard).await?; debug!( chunk_count, chunk_payload_bytes, @@ -3235,128 +3180,167 @@ where ); return Ok(()); } + if trailer_count == MAX_CHUNK_TRAILER_FIELDS { + return Err(miette!( + "Chunk trailers exceed {MAX_CHUNK_TRAILER_FIELDS} fields" + )); + } trailer_count += 1; - if pos > RELAY_BUF_SIZE * 4 && forwarded_len >= pos { - parse_buf.drain(..pos); - forwarded_len -= pos; - pos = 0; + if input.pending_len() >= RELAY_BUF_SIZE { + input.flush_pending(writer, generation_guard).await?; } } } - // Ensure the full chunk payload + trailing CRLF is available. - let chunk_end = pos - .checked_add(chunk_size) - .ok_or_else(|| miette!("Chunk size overflow"))?; - let chunk_with_crlf_end = chunk_end - .checked_add(2) - .ok_or_else(|| miette!("Chunk size overflow"))?; - - relay_chunked_until_len( - reader, - &mut read_buf, - &mut parse_buf, - chunk_with_crlf_end, - generation_guard, - "Chunked body ended mid-chunk", - ) - .await?; - if &parse_buf[chunk_end..chunk_with_crlf_end] != b"\r\n" { + input + .consume_exact( + reader, + writer, + chunk_size, + generation_guard, + "Chunked body ended mid-chunk", + ) + .await?; + let terminator = [ + input + .read_byte(reader, "Chunked body ended before chunk terminator") + .await?, + input + .read_byte(reader, "Chunked body ended before chunk terminator") + .await?, + ]; + if terminator != *b"\r\n" { return Err(miette!("Chunk missing terminating CRLF")); } - pos = chunk_with_crlf_end; - flush_validated_chunked_bytes( - writer, - &parse_buf, - &mut forwarded_len, - pos, - true, - generation_guard, - ) - .await?; + input.flush_pending(writer, generation_guard).await?; chunk_count += 1; chunk_payload_bytes = chunk_payload_bytes.saturating_add(chunk_size); - - // Keep parser memory bounded for long streams. - if pos > RELAY_BUF_SIZE * 4 && forwarded_len >= pos { - parse_buf.drain(..pos); - forwarded_len -= pos; - pos = 0; - } } } -/// Read only the bytes needed to reach `target_len`. -/// -/// The connection-scoped buffered reader may fetch more data from the socket, -/// but this parser consumes only the current body. Any read-ahead remains in -/// that reader for the next request's policy decision. -async fn relay_chunked_until_len( - reader: &mut R, - read_buf: &mut [u8; RELAY_BUF_SIZE], - parse_buf: &mut Vec, - target_len: usize, - generation_guard: Option<&PolicyGenerationGuard>, - eof_message: &'static str, -) -> Result<()> -where - R: AsyncRead + Unpin, -{ - while parse_buf.len() < target_len { - let remaining = target_len - parse_buf.len(); - let to_read = remaining.min(read_buf.len()); - let n = reader - .read(&mut read_buf[..to_read]) - .await - .into_diagnostic()?; - if n == 0 { - return Err(miette!(eof_message)); +struct ChunkedRelayInput<'a> { + already_forwarded: &'a [u8], + already_forwarded_pos: usize, + pending: Vec, +} + +impl<'a> ChunkedRelayInput<'a> { + fn new(already_forwarded: &'a [u8]) -> Self { + Self { + already_forwarded, + already_forwarded_pos: 0, + pending: Vec::with_capacity(RELAY_BUF_SIZE), } - if let Some(guard) = generation_guard { - guard.ensure_current()?; + } + + fn pending_len(&self) -> usize { + self.pending.len() + } + + async fn read_line( + &mut self, + reader: &mut R, + eof_message: &'static str, + limit_message: &'static str, + ) -> Result> + where + R: AsyncRead + Unpin, + { + let mut line = Vec::new(); + loop { + if line.len() == MAX_CHUNK_LINE_BYTES { + return Err(miette!(limit_message)); + } + line.push(self.read_byte(reader, eof_message).await?); + if line.ends_with(b"\r\n") { + line.truncate(line.len() - 2); + return Ok(line); + } } - parse_buf.extend_from_slice(&read_buf[..n]); } - Ok(()) -} -/// Forward complete, validated chunk framing. -/// -/// Size lines may be coalesced with their payload, but every completed chunk is -/// forwarded immediately so streaming request and response bodies make -/// progress without waiting for the terminal chunk. This writes by validated -/// framing units rather than once per framing byte. -async fn flush_validated_chunked_bytes( - writer: &mut W, - parse_buf: &[u8], - forwarded_len: &mut usize, - validated_len: usize, - force: bool, - generation_guard: Option<&PolicyGenerationGuard>, -) -> Result<()> -where - W: AsyncWrite + Unpin, -{ - let pending = validated_len.saturating_sub(*forwarded_len); - if pending == 0 || (!force && pending < RELAY_BUF_SIZE) { - return Ok(()); + async fn consume_exact( + &mut self, + reader: &mut R, + writer: &mut W, + mut remaining: usize, + generation_guard: Option<&PolicyGenerationGuard>, + eof_message: &'static str, + ) -> Result<()> + where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin, + { + if self.already_forwarded_pos < self.already_forwarded.len() { + let available = self.already_forwarded.len() - self.already_forwarded_pos; + let consumed = available.min(remaining); + self.already_forwarded_pos += consumed; + remaining -= consumed; + } + + while remaining > 0 { + if self.pending.len() >= RELAY_BUF_SIZE { + self.flush_pending(writer, generation_guard).await?; + } + let available = RELAY_BUF_SIZE - self.pending.len(); + let to_read = remaining.min(available); + let start = self.pending.len(); + self.pending.resize(start + to_read, 0); + let read = reader + .read(&mut self.pending[start..]) + .await + .into_diagnostic()?; + if read == 0 { + self.pending.truncate(start); + return Err(miette!(eof_message)); + } + self.pending.truncate(start + read); + if let Some(guard) = generation_guard { + guard.ensure_current()?; + } + remaining -= read; + } + Ok(()) } - if let Some(guard) = generation_guard { - guard.ensure_current()?; + + async fn read_byte(&mut self, reader: &mut R, eof_message: &'static str) -> Result + where + R: AsyncRead + Unpin, + { + if self.already_forwarded_pos < self.already_forwarded.len() { + let byte = self.already_forwarded[self.already_forwarded_pos]; + self.already_forwarded_pos += 1; + return Ok(byte); + } + let byte = match reader.read_u8().await { + Ok(byte) => byte, + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => { + return Err(miette!(eof_message)); + } + Err(error) => return Err(error).into_diagnostic(), + }; + self.pending.push(byte); + Ok(byte) } - writer - .write_all(&parse_buf[*forwarded_len..validated_len]) - .await - .into_diagnostic()?; - *forwarded_len = validated_len; - Ok(()) -} -fn find_crlf(buf: &[u8], start: usize) -> Option { - buf.get(start..)? - .windows(2) - .position(|w| w == b"\r\n") - .map(|offset| start + offset) + async fn flush_pending( + &mut self, + writer: &mut W, + generation_guard: Option<&PolicyGenerationGuard>, + ) -> Result<()> + where + W: AsyncWrite + Unpin, + { + if self.pending.is_empty() { + return Ok(()); + } + if let Some(guard) = generation_guard { + guard.ensure_current()?; + } + writer.write_all(&self.pending).await.into_diagnostic()?; + self.pending.clear(); + Ok(()) + } } fn validate_websocket_response( @@ -4008,6 +3992,81 @@ mod tests { } } + #[tokio::test] + async fn relay_chunked_streams_max_sized_incomplete_chunk_without_retaining_payload() { + let payload_len = RELAY_BUF_SIZE * 32; + let mut wire = format!("{:x}\r\n", usize::MAX).into_bytes(); + wire.resize(wire.len() + payload_len, b'a'); + let wire_len = wire.len(); + let mut reader = CountingReader::new(wire); + let mut writer = tokio::io::sink(); + + let error = relay_chunked(&mut reader, &mut writer, &[], None) + .await + .expect_err("an incomplete chunk must fail"); + + assert!(error.to_string().contains("ended mid-chunk")); + assert_eq!( + reader.position, wire_len, + "the relay must stream the payload instead of rejecting or accumulating the declared chunk" + ); + } + + #[tokio::test] + async fn relay_chunked_rejects_oversized_chunk_size_line() { + let mut wire = b"1;".to_vec(); + wire.resize(MAX_CHUNK_LINE_BYTES + 1, b'x'); + wire.extend_from_slice(b"\r\na\r\n0\r\n\r\n"); + let mut reader = CountingReader::new(wire); + let mut writer = tokio::io::sink(); + + let error = relay_chunked(&mut reader, &mut writer, &[], None) + .await + .expect_err("an oversized chunk-size line must fail"); + + assert!(error.to_string().contains("Chunk-size line exceeds")); + } + + #[tokio::test] + async fn relay_chunked_rejects_oversized_trailer_block() { + let mut wire = b"0\r\n".to_vec(); + for name in [b'a', b'b'] { + wire.extend_from_slice(&[name, b':']); + wire.resize(wire.len() + MAX_HEADER_BYTES / 2, b'x'); + wire.extend_from_slice(b"\r\n"); + } + wire.extend_from_slice(b"\r\n"); + let mut reader = CountingReader::new(wire); + let mut writer = tokio::io::sink(); + + let error = relay_chunked(&mut reader, &mut writer, &[], None) + .await + .expect_err("an oversized trailer block must fail"); + + assert!(error.to_string().contains("Chunk trailers exceed")); + } + + #[tokio::test] + async fn relay_chunked_rejects_excessive_trailer_fields() { + let mut wire = b"0\r\n".to_vec(); + for _ in 0..=MAX_CHUNK_TRAILER_FIELDS { + wire.extend_from_slice(b"x: y\r\n"); + } + wire.extend_from_slice(b"\r\n"); + let mut reader = CountingReader::new(wire); + let mut writer = tokio::io::sink(); + + let error = relay_chunked(&mut reader, &mut writer, &[], None) + .await + .expect_err("too many trailer fields must fail"); + + assert!( + error + .to_string() + .contains("Chunk trailers exceed 128 fields") + ); + } + fn write_header(name: &str, value: &str, on_existing: ExistingHeaderAction) -> HeaderMutation { HeaderMutation { operation: Some(header_mutation::Operation::Write( From 293fab75d4276406aa0c3463989800120fe85a79 Mon Sep 17 00:00:00 2001 From: Akram Ben Aissi Date: Tue, 22 Sep 2026 20:37:19 +0000 Subject: [PATCH 2/9] fix(sandbox): support kernels < 5.19 via seccomp WAIT_KILLABLE_RECV fallback (#3420) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sandbox): fall back to plain seccomp listener when WAIT_KILLABLE_RECV is unavailable SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV was added in Linux 5.19. On older kernels (for example RHEL 9.x / 5.14 nodes such as RHCOS on OpenShift) the flag is rejected with EINVAL, which made the capability-free sandbox fail to start during the notification probe with "notification launcher disappeared". Install the notification listener with WAIT_KILLABLE_RECV when the kernel supports it and fall back to a plain NEW_LISTENER on EINVAL. The fallback listener records wait_killable_recv = false: its notification receive is uninterruptible, but the sandbox is otherwise fully functional. Signed-off-by: Akram * fix(sandbox): address review — legacy read-only listener mode + cancellation invariant Follow-up to the PR review (GATOR-de00bfcc-01 / mrunalp): make the < 5.19 fallback cancellation-safe instead of racing broker writes. - Record an explicit ListenerMode (Killable vs LegacyReadOnly); add writes_disabled()/mode() and emit the selected mode in qualification output (seccomp_listener_mode). - Centralize task-memory output writes behind NotificationListener:: write_task_output; in LegacyReadOnly mode getpeername, accept/accept4 with a non-null address, and sendmmsg length write-backs fail closed with EOPNOTSUPP. accept with a null address, socket/connect/bind/listen/sendto/sendmsg keep working (copied inputs, scalar responses, atomic ADDFD_SEND). - Enforce the launch invariant `cancellation || task_memory_writes_disabled` in SandboxConfirmEvidence::validate() rather than dropping cancellation unconditionally; add task_memory_writes_disabled to SeccompEvidence. - Add per-path fail-closed tests (write_task_output, write_socket_addr, a real plain listener installed on a modern kernel) and confirmation-invariant tests. - Correct the flag-semantics comments and document both modes plus the reduced legacy syscall compatibility in architecture/sandbox.md. Signed-off-by: Akram * docs: document legacy read-only sandbox mode on kernels before 5.19 Kernels without SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV (< 5.19, e.g. RHEL 9.x / RHCOS 5.14) run the sandbox in a legacy read-only cancellation mode where the broker fails closed with EOPNOTSUPP on the mediated operations that write results back into workload memory (getpeername, accept/accept4 with a non-null address, sendmmsg length write-backs). Document this observable behavior and its syscall limitations in the public Fern docs: the support-matrix kernel requirements and the OpenShift runtime guidance. Signed-off-by: Akram * style(sandbox): satisfy rustfmt and clippy doc_markdown Match the pinned rustfmt (Rust 1.95.0) line-wrapping for the write_task_output call, and backtick `legacy_read_only` in the qualification-report doc comment so clippy::doc_markdown (-D warnings) passes. Signed-off-by: Akram * fix(sandbox): migrate task_memory_writes_disabled into backend protocol Add the `task_memory_writes_disabled` field to `SeccompEvidence` in the backend protocol contract and relax the validation from requiring `cancellation` to accepting `cancellation || task_memory_writes_disabled`. This completes the rebase migration missed by the isolation-interface refactor: the sandbox reports this field but the backend struct lacked it, and the validator rejected every pre-5.19 legacy listener. Signed-off-by: Akram --------- Signed-off-by: Akram --- architecture/sandbox.md | 28 +++- .../src/linux/seccomp_notify.rs | 156 +++++++++++++++++- .../src/boundary_protocol.rs | 20 ++- .../openshell-sandbox-backend/src/runtime.rs | 1 + .../openshell-sandbox/src/boundary_server.rs | 1 + crates/openshell-sandbox/src/main.rs | 14 ++ .../openshell-sandbox/src/network_broker.rs | 44 ++++- docs/kubernetes/openshift.mdx | 15 ++ docs/reference/support-matrix.mdx | 25 +++ 9 files changed, 285 insertions(+), 19 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index e8412a43f2..4e7ddd0385 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -205,9 +205,31 @@ The sandbox reserves `SIGUSR2` with a non-restarting no-op handler for these broker threads; startup rejects a conflicting handler. This signal disposition is process-global kernel state, while registrations and cancellation state are owned by the broker. Workload exec resets the caught handler to its default. -This sandbox runtime requires Linux 6.2 or newer for Landlock ABI v3 and treats -`SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` as mandatory so cancelled -notifications cannot race task-memory writes. +This sandbox runtime requires Landlock ABI v3 (Linux 6.2, or an equivalent +vendor backport). The seccomp listener is installed in one of two cancellation +modes, and the launch confirmation enforces the invariant +`cancellation || task_memory_writes_disabled`: + +- **Killable** (`SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV`, Linux 5.19+): the + notified workload thread waits kill-only, so a non-fatal signal cannot resume + a mediated syscall between notification validation and the broker's result + write. Full mediation, including task-memory output writes. +- **LegacyReadOnly** (kernels < 5.19, e.g. RHEL 9.x / 5.14): the flag is + unavailable (`EINVAL`), so the listener falls back to a plain notifier and the + broker refuses every task-memory *output* write to stay cancellation-safe. + Concretely, in this mode `getpeername`, `accept`/`accept4` **with a non-null + peer-address argument**, and `sendmmsg` paths that write per-message lengths + fail closed with `EOPNOTSUPP`. `accept` with a null address, and socket + creation, `connect`, `bind`, `listen`, `sendto`, and `sendmsg` continue to + work — they use copied inputs, scalar responses, or atomic `ADDFD_SEND`, none + of which write into workload memory. Some server workloads whose accept + wrappers request the peer address will therefore not run until the kernel + provides `WAIT_KILLABLE_RECV` (a distribution backport); outbound-oriented + workloads are unaffected. + +Input mediation, DNS/TCP authorization, and outer-fence enforcement are +identical in both modes. The selected mode is emitted in the sandbox +qualification output (`seccomp_listener_mode`). DNS uses an exact sandbox-local resolver at `127.0.0.53:53`. The driver sets the nameserver and permits an unprivileged bind to port 53. UDP and TCP DNS requests diff --git a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs index 7d24dfa4d9..4f86844d10 100644 --- a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs +++ b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs @@ -177,6 +177,34 @@ impl NotificationProbeReport { } } +/// Cancellation posture a listener was installed with. +/// +/// `WAIT_KILLABLE_RECV` (Linux 5.19+) keeps the notified workload thread in a +/// kill-only wait so a non-fatal signal cannot resume the mediated syscall +/// after the broker has validated the notification. A plain listener has no +/// such guarantee, so it runs read-only: the broker must refuse every +/// task-memory *output* write to stay cancellation-safe. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ListenerMode { + /// Modern kernel: `WAIT_KILLABLE_RECV` active; full mediation including + /// task-memory output writes. + Killable, + /// Legacy kernel (< 5.19): plain listener; task-memory output writes are + /// disabled so a resumed syscall cannot race a broker write. + LegacyReadOnly, +} + +impl ListenerMode { + /// Stable identifier for qualification output and diagnostics. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Killable => "killable", + Self::LegacyReadOnly => "legacy_read_only", + } + } +} + /// Owned listener returned by `SECCOMP_FILTER_FLAG_NEW_LISTENER`. pub struct NotificationListener { fd: OwnedFd, @@ -184,6 +212,17 @@ pub struct NotificationListener { } impl NotificationListener { + /// Construct a listener from an already-owned notification descriptor in a + /// specific mode. Intended for tests that must exercise the legacy + /// read-only fail-closed paths without a `< 5.19` kernel. + #[must_use] + pub fn from_fd_with_mode(fd: OwnedFd, mode: ListenerMode) -> Self { + Self { + fd, + wait_killable_recv: matches!(mode, ListenerMode::Killable), + } + } + /// Raw listener descriptor for readiness integration and diagnostics. #[must_use] pub fn as_raw_fd(&self) -> RawFd { @@ -196,6 +235,23 @@ impl NotificationListener { self.wait_killable_recv } + /// The cancellation mode this listener was installed with. + #[must_use] + pub fn mode(&self) -> ListenerMode { + if self.wait_killable_recv { + ListenerMode::Killable + } else { + ListenerMode::LegacyReadOnly + } + } + + /// Whether broker task-memory output writes are disabled for this listener. + /// True exactly in `LegacyReadOnly` mode (no `WAIT_KILLABLE_RECV`). + #[must_use] + pub fn writes_disabled(&self) -> bool { + matches!(self.mode(), ListenerMode::LegacyReadOnly) + } + /// Receive the next kernel notification. pub fn receive(&self) -> io::Result { let mut raw = RawNotification::default(); @@ -223,6 +279,34 @@ impl NotificationListener { Ok(()) } + /// Write broker-produced output into the notified task's memory, closing + /// the validation-to-write race that a plain listener cannot. + /// + /// In `Killable` mode `WAIT_KILLABLE_RECV` keeps the notified workload + /// thread in a kill-only wait, so a non-fatal signal cannot resume the + /// mediated syscall between `validate_id` and this write. In + /// `LegacyReadOnly` mode (kernels < 5.19) there is no such guarantee: a + /// resumed syscall could repurpose the target buffer while the privileged + /// broker writes through the captured tid and pointer — via `/proc//mem` + /// even into pages the workload has since made read-only. There is no way to + /// close that window without the flag, so this fails closed (`EOPNOTSUPP`) + /// rather than racing. Callers must route every task-memory *output* write + /// through this method; input reads never write workload memory and are + /// unaffected. + pub fn write_task_output( + &self, + id: u64, + tid: u32, + address: u64, + data: &[u8], + ) -> io::Result<()> { + if self.writes_disabled() { + return Err(io::Error::from_raw_os_error(libc::EOPNOTSUPP)); + } + self.validate_id(id)?; + crate::linux::task_memory::write_exact(tid, address, data) + } + /// Return a successful scalar result to the notifying syscall. pub fn respond_value(&self, id: u64, value: i64) -> io::Result<()> { self.validate_id(id)?; @@ -330,16 +414,22 @@ pub fn install_listener(syscalls: &[i64]) -> io::Result { verify_notification_sizes()?; set_no_new_privileges()?; - install_listener_with_flags(syscalls, true).map_err(|error| { - if error.raw_os_error() == Some(libc::EINVAL) { - io::Error::new( - io::ErrorKind::Unsupported, - "seccomp WAIT_KILLABLE_RECV is required (Linux 5.19 or newer)", - ) - } else { - error + // WAIT_KILLABLE_RECV (Linux 5.19+) keeps the *notified workload thread* in + // a kill-only wait while the broker services its syscall, so a non-fatal + // signal cannot resume the syscall and repurpose its buffers underneath a + // pending broker write. Kernels older than 5.19 (for example RHEL 9.x / + // 5.14 nodes) reject the flag with EINVAL. Rather than refusing to start + // there, fall back to a plain listener so the sandbox boots; the resulting + // listener records `wait_killable_recv = false`, and the broker then fails + // closed on every task-memory output write (see `write_task_output`) + // instead of racing them. Input mediation is unaffected. + match install_listener_with_flags(syscalls, true) { + Ok(listener) => Ok(listener), + Err(error) if error.raw_os_error() == Some(libc::EINVAL) => { + install_listener_with_flags(syscalls, false) } - }) + Err(error) => Err(error), + } } /// Install the capability-free workload networking listener on the calling @@ -935,4 +1025,52 @@ mod tests { .expect_err("zero errno must fail"); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); } + + #[test] + fn plain_listener_fails_closed_on_output_write() { + // A LegacyReadOnly listener (kernels < 5.19) must refuse every + // task-memory output write rather than race a resumed syscall. The + // guard short-circuits before touching the descriptor or workload + // memory, so a dup of stderr is a sufficient stand-in. + // SAFETY: dup takes one valid descriptor and returns a new descriptor + // or a negative error without modifying memory. + let duplicated = unsafe { libc::dup(libc::STDERR_FILENO) }; + assert!(duplicated >= 0, "duplicate stderr for validation test"); + // SAFETY: successful dup returned a new owned descriptor. + let listener = NotificationListener::from_fd_with_mode( + unsafe { OwnedFd::from_raw_fd(duplicated) }, + ListenerMode::LegacyReadOnly, + ); + assert!(listener.writes_disabled()); + assert_eq!(listener.mode(), ListenerMode::LegacyReadOnly); + let error = listener + .write_task_output(1, 0, 0, &[0_u8; 4]) + .expect_err("plain listener must reject output writes"); + assert_eq!(error.raw_os_error(), Some(libc::EOPNOTSUPP)); + } + + #[test] + fn real_plain_listener_disables_output_writes() { + // Deliberately install a plain NEW_LISTENER (no WAIT_KILLABLE_RECV) + // even on a modern CI kernel and prove the broker write path fails + // closed on the real listener object. + set_no_new_privileges().expect("no_new_privs for listener install"); + let listener = install_listener_with_flags(&[libc::SYS_getppid], false) + .expect("install plain listener"); + assert_eq!(listener.mode(), ListenerMode::LegacyReadOnly); + assert!(listener.writes_disabled()); + let error = listener + .write_task_output(1, 0, 0, &[0_u8; 4]) + .expect_err("plain listener must reject output writes"); + assert_eq!(error.raw_os_error(), Some(libc::EOPNOTSUPP)); + } + + #[test] + fn killable_listener_enables_output_writes() { + set_no_new_privileges().expect("no_new_privs for listener install"); + let listener = install_listener_with_flags(&[libc::SYS_getppid], true) + .expect("install killable listener"); + assert_eq!(listener.mode(), ListenerMode::Killable); + assert!(!listener.writes_disabled()); + } } diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index 2c56fd1ed1..a5edb2accc 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -82,6 +82,7 @@ pub struct SeccompEvidence { pub task_memory_read: bool, pub task_memory_write: bool, pub cancellation: bool, + pub task_memory_writes_disabled: bool, } /// Mechanism-specific audit evidence for the native Linux sandbox adapter. @@ -129,7 +130,7 @@ impl NativeLinuxSandboxAuditEvidence { && self.seccomp.proc_fd_identity && self.seccomp.task_memory_read && self.seccomp.task_memory_write - && self.seccomp.cancellation + && (self.seccomp.cancellation || self.seccomp.task_memory_writes_disabled) && self.landlock_abi >= 3 && self.landlock_allow_deny && self.udp_dns_round_trip @@ -1408,6 +1409,7 @@ mod tests { task_memory_read: true, task_memory_write: true, cancellation: true, + task_memory_writes_disabled: false, }, landlock_abi: 6, landlock_allow_deny: true, @@ -1438,6 +1440,22 @@ mod tests { assert!(!audit.properties().egress_interception.enforced); } + #[test] + fn audit_evidence_accepts_legacy_read_only_listener() { + let mut audit = complete_audit_evidence(); + audit.seccomp.cancellation = false; + audit.seccomp.task_memory_writes_disabled = true; + assert!(audit.validate().is_ok()); + } + + #[test] + fn audit_evidence_rejects_plain_listener_with_writes_enabled() { + let mut audit = complete_audit_evidence(); + audit.seccomp.cancellation = false; + audit.seccomp.task_memory_writes_disabled = false; + assert!(audit.validate().is_err()); + } + #[test] fn binary_identity_wire_rejects_ambiguous_or_invalid_shapes() { for encoded in [ diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 06891d8aa8..39bdd77aba 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -2623,6 +2623,7 @@ mod tests { task_memory_read: true, task_memory_write: true, cancellation: true, + task_memory_writes_disabled: false, }, landlock_abi: 3, landlock_allow_deny: true, diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index f126ec6d58..8fbb79b888 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -4329,6 +4329,7 @@ mod linux { task_memory_read: true, task_memory_write: true, cancellation: true, + task_memory_writes_disabled: false, }, landlock_abi: 6, landlock_allow_deny: true, diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index a2c68fbcf7..d6d2389e77 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -146,6 +146,11 @@ struct QualificationReport { tcp_allow_round_trip: bool, tcp_deny_round_trip: bool, wait_killable_recv: bool, + /// Selected seccomp listener cancellation mode: `killable` (>= 5.19) or + /// `legacy_read_only` (< 5.19, broker output writes disabled). + seccomp_listener_mode: &'static str, + /// Whether the broker disables task-memory output writes (legacy mode). + task_memory_writes_disabled: bool, } #[cfg(target_os = "linux")] @@ -237,6 +242,12 @@ fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, Qualifi tcp_allow_round_trip: true, tcp_deny_round_trip: true, wait_killable_recv: notification.wait_killable_recv, + seccomp_listener_mode: if notification.wait_killable_recv { + "killable" + } else { + "legacy_read_only" + }, + task_memory_writes_disabled: !notification.wait_killable_recv, }; let qualification = openshell_sandbox::RuntimeQualification { seccomp: openshell_sandbox_backend::boundary_protocol::SeccompEvidence { @@ -249,6 +260,9 @@ fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, Qualifi task_memory_read: notification.task_memory_copy(), task_memory_write: notification.task_memory_copy(), cancellation: notification.wait_killable_recv, + // Legacy plain listener (< 5.19) disables broker output writes; + // satisfies the `cancellation || writes_disabled` launch invariant. + task_memory_writes_disabled: !notification.wait_killable_recv, }, landlock_abi, landlock_allow_deny: true, diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index d093ebf199..744ce89677 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -1379,8 +1379,8 @@ fn classify_send( if let Some(length_address) = message.result_length_address { let length = u32::try_from(message.data.len()) .map_err(|_| io::Error::from_raw_os_error(libc::EMSGSIZE))?; - listener.validate_id(notification.id)?; - task_memory::write_exact( + listener.write_task_output( + notification.id, notification.tid, length_address, &length.to_ne_bytes(), @@ -1733,6 +1733,15 @@ fn write_socket_addr( length_address: u64, value: SocketAddr, ) -> io::Result<()> { + // A LegacyReadOnly listener (kernels < 5.19) cannot safely write into + // workload memory: without WAIT_KILLABLE_RECV the notified accept/ + // getpeername could resume and repurpose these buffers between validation + // and the broker write. Fail closed before reading or writing anything, so + // this address-writing path is inert in legacy mode. Callers that pass a + // null address argument (accept with a null peer address) never reach here. + if listener.writes_disabled() { + return Err(io::Error::from_raw_os_error(libc::EOPNOTSUPP)); + } let mut supplied_length = [0_u8; size_of::()]; task_memory::read_exact(tid, length_address, &mut supplied_length)?; let supplied_length = libc::socklen_t::from_ne_bytes(supplied_length); @@ -1740,12 +1749,15 @@ fn write_socket_addr( let copied = usize::try_from(supplied_length) .unwrap_or(0) .min(bytes.len()); - listener.validate_id(notification_id)?; if copied != 0 { - task_memory::write_exact(tid, address, &bytes[..copied])?; + listener.write_task_output(notification_id, tid, address, &bytes[..copied])?; } - listener.validate_id(notification_id)?; - task_memory::write_exact(tid, length_address, &actual_length.to_ne_bytes()) + listener.write_task_output( + notification_id, + tid, + length_address, + &actual_length.to_ne_bytes(), + ) } fn sockaddr_bytes(address: SocketAddr) -> io::Result<(Vec, libc::socklen_t)> { @@ -1814,6 +1826,7 @@ fn error_to_errno(error: &io::Error) -> i32 { #[cfg(test)] mod tests { use super::*; + use openshell_isolation_interface::linux::seccomp_notify::ListenerMode; use std::io::{Read as _, Write as _}; use std::os::unix::net::{UnixListener, UnixStream}; @@ -1912,6 +1925,25 @@ mod tests { ))); } + #[test] + fn legacy_listener_rejects_socket_addr_write() { + // accept-with-address and getpeername both route through + // write_socket_addr; on a LegacyReadOnly listener the path must fail + // closed (EOPNOTSUPP) before any task-memory access. + // SAFETY: dup returns a new descriptor or a negative error. + let dup = unsafe { libc::dup(libc::STDERR_FILENO) }; + assert!(dup >= 0, "dup stderr"); + let listener = NotificationListener::from_fd_with_mode( + // SAFETY: successful dup returned a new owned descriptor. + unsafe { OwnedFd::from_raw_fd(dup) }, + ListenerMode::LegacyReadOnly, + ); + let peer: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let error = write_socket_addr(&listener, 1, 0, 0, 0, peer) + .expect_err("legacy listener must reject socket-address writes"); + assert_eq!(error.raw_os_error(), Some(libc::EOPNOTSUPP)); + } + #[test] fn relay_rejects_descriptor_replaced_after_policy_decision() { let metadata = SocketMetadata { diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index aaad7f6c53..39b0a4bf14 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -19,6 +19,21 @@ process to install a nested seccomp user-notification filter and use Landlock. OpenShell fails sandbox startup when either capability-free runtime probe fails. +## Node kernel and legacy read-only mode + +OpenShift nodes run RHCOS, which currently ships a RHEL 9.x kernel (5.14). That +kernel predates `SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` (Linux 5.19), so the +sandbox starts in a reduced **legacy read-only** cancellation mode. Isolation is +unchanged, but the broker fails closed with `EOPNOTSUPP` on the mediated +operations that write results back into workload memory — `getpeername`, +`accept`/`accept4` with a non-null peer-address argument, and `sendmmsg` +per-message length write-backs. Outbound-oriented workloads run unchanged; +server workloads that read the peer address on accept need a node kernel with +`WAIT_KILLABLE_RECV` (Linux 5.19+, or a distribution backport). See the +[support matrix](/reference/support-matrix#legacy-read-only-mode-kernels-before-linux-519) +for the full behavior; the selected mode is reported as `seccomp_listener_mode` +in the sandbox qualification output. + ## Prerequisites - OpenShift 4.x cluster with `oc` configured. diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index c587ee6a91..8f49dcc945 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -103,12 +103,37 @@ when it runs inside a container or microVM: | -------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | [Landlock LSM](https://docs.kernel.org/security/landlock.html) | Required | ABI 3 or newer, introduced in Linux 6.2, with Landlock enabled. The mandatory baseline protects private channel and bootstrap files, including against truncation. A filesystem policy's `best_effort` setting never disables this baseline. | | seccomp | Required | Nested user-notification filters and atomic `SECCOMP_IOCTL_NOTIF_ADDFD` with `SECCOMP_ADDFD_FLAG_SEND`, usable under the runtime's existing seccomp profile without added capabilities. The sandbox actively probes these operations before admitting the workload. | +| seccomp `WAIT_KILLABLE_RECV` | Recommended (Linux 5.19+) | Keeps a notified workload thread in a kill-only wait so the broker can safely write mediated results into workload memory. Without it (kernels < 5.19, for example RHEL 9.x / RHCOS 5.14) the sandbox still starts, in a reduced **legacy read-only** mode described below. | A kernel version alone does not establish support. A disabled Landlock LSM or a runtime profile that blocks the required seccomp operations causes launch to fail closed. An upstream Linux 6.2 or newer kernel provides the required Landlock ABI; distribution backports must pass the same active qualification. +### Legacy read-only mode (kernels before Linux 5.19) + +`SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` was added in Linux 5.19. On older +kernels — notably RHEL 9.x and RHCOS, which ship a 5.14 kernel — the sandbox +cannot install a kill-only listener, so it falls back to a plain listener and +runs in a **legacy read-only** cancellation mode. The sandbox starts and +enforces the full isolation boundary (Landlock, the outer NetworkPolicy fence, +DNS and TCP authorization); the only difference is that the broker refuses the +mediated operations that write results back into workload memory, failing them +closed with `EOPNOTSUPP`: + +- `getpeername`; +- `accept` / `accept4` **when a non-null peer-address argument is supplied** + (a null address argument still works); +- `sendmmsg` paths that write per-message lengths back to the caller. + +Socket creation, `connect`, `bind`, `listen`, `sendto`, and `sendmsg` are +unaffected. Outbound-oriented workloads generally run unchanged; server +workloads whose accept wrappers request the peer address will see `EOPNOTSUPP` +until the node runs a kernel that provides `WAIT_KILLABLE_RECV` (Linux 5.19+, or +a distribution backport). The selected mode is reported in the sandbox +qualification output as `seccomp_listener_mode` (`killable` or +`legacy_read_only`). + On macOS, these kernel modules run inside the Docker Desktop Linux VM, not on the host kernel. ## Agent Compatibility From f8002d19ad2f948abf48bd2f5ca4f8ebd388e3c8 Mon Sep 17 00:00:00 2001 From: krishicks Date: Tue, 22 Sep 2026 20:55:04 +0000 Subject: [PATCH 3/9] fix(e2e): repair the credential driver test (#3565) The credential driver e2e has not passed end to end, and the disabled kubernetes-credential-drivers CI lane hid three problems. The test broke when the JSON format of provider list changed. Continuation-token pagination (#3249) changed provider list --output json from a bare array of providers to an object with next_page_token and a providers array. The test still parsed the output as an array, so it failed before checking either storage backend. Read the providers array from the new object instead. Its sandbox name was about 58 characters, but sandbox names are DNS-routable and limited to 19, so sandbox creation was rejected. Build a short unique name instead. The sandbox guard deletes its sandbox from a detached thread on drop, so the test deleted the provider while the sandbox still existed. The gateway rejects deleting a provider that is attached to a sandbox, the test ignored that error, and the credential Secret remained. Delete the sandbox explicitly before returning from the sandbox check. Signed-off-by: Kris Hicks --- e2e/rust/tests/credential_drivers.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/e2e/rust/tests/credential_drivers.rs b/e2e/rust/tests/credential_drivers.rs index 6c4088a2f3..3901de1c15 100644 --- a/e2e/rust/tests/credential_drivers.rs +++ b/e2e/rust/tests/credential_drivers.rs @@ -182,9 +182,11 @@ async fn provider_identity(provider_name: &str) -> Result = serde_json::from_str(&clean) + let listing: serde_json::Value = serde_json::from_str(&clean) .map_err(|err| format!("failed to parse provider list JSON: {err}\n{clean}"))?; - let provider = providers + let provider = listing["providers"] + .as_array() + .ok_or_else(|| format!("provider list JSON has no providers array:\n{clean}"))? .iter() .find(|provider| provider["name"].as_str() == Some(provider_name)) .ok_or_else(|| format!("provider '{provider_name}' was not returned by provider list"))?; @@ -225,7 +227,7 @@ async fn assert_provider_placeholder_available_in_sandbox( sandbox_name: &str, secret_value: &str, ) -> Result<(), String> { - let guard = SandboxGuard::create(&[ + let mut guard = SandboxGuard::create(&[ "--name", sandbox_name, "--provider", @@ -239,6 +241,9 @@ async fn assert_provider_placeholder_available_in_sandbox( ]) .await?; let clean = strip_ansi(&guard.create_output); + // Delete the sandbox before returning: the gateway refuses to delete a + // provider that is still attached to a sandbox. + guard.cleanup().await; if !contains_placeholder_for_env_key(&clean, CREDENTIAL_KEY) { return Err(format!( "sandbox {sandbox_name} did not receive provider credential placeholder:\n{clean}" @@ -374,7 +379,12 @@ async fn provider_credentials_are_stored_in_configured_backend() { let suffix = unique_suffix(); let driver_slug = driver.replace('-', ""); let provider_name = format!("cred-storage-{driver_slug}-{suffix}"); - let sandbox_name = format!("cred-storage-sandbox-{driver_slug}-{suffix}"); + // Sandbox names are DNS-routable and limited to 19 characters. + let sandbox_name = format!( + "cred-{}-{}", + &driver_slug[..1], + &suffix[suffix.len() - 10..] + ); let secret_value = format!("example-e2e-{driver_slug}-{suffix}"); delete_provider(&provider_name).await; From 1e34e8c5768afb8e0a841c2a4320fa5f32240432 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 21:22:30 +0000 Subject: [PATCH 4/9] fix(drivers): require admission labels for external resources (#3538) * fix(drivers): require admission labels for external resources Signed-off-by: Drew Newberry * fix(drivers): address resource admission review findings Signed-off-by: Drew Newberry * fix(core): reserve driver-owned admission labels Signed-off-by: Drew Newberry * fix(core): clarify workspace admission label Signed-off-by: Drew Newberry * fix(drivers): clarify resource admission failures Signed-off-by: Drew Newberry * test(e2e): configure resource admission fixtures Signed-off-by: Drew Newberry * fix(kubernetes): retry forbidden admission lookups Signed-off-by: Drew Newberry * fix(e2e): preserve external driver admission defaults Signed-off-by: Drew Newberry --------- Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 7 + Cargo.lock | 1 + architecture/compute-runtimes.md | 29 +- crates/openshell-cli/src/main.rs | 8 +- crates/openshell-core/src/lib.rs | 1 + .../openshell-core/src/resource_admission.rs | 415 +++++++++++ crates/openshell-driver-docker/README.md | 9 +- crates/openshell-driver-docker/src/lib.rs | 227 +++++- crates/openshell-driver-docker/src/main.rs | 10 +- crates/openshell-driver-docker/src/tests.rs | 143 ++++ crates/openshell-driver-kubernetes/README.md | 7 + .../openshell-driver-kubernetes/src/config.rs | 7 + .../openshell-driver-kubernetes/src/driver.rs | 668 +++++++++++++++++- crates/openshell-driver-kubernetes/src/lib.rs | 1 + .../openshell-driver-kubernetes/src/main.rs | 9 + .../src/resource_admission.rs | 515 ++++++++++++++ crates/openshell-driver-mxc/README.md | 6 + crates/openshell-driver-mxc/src/driver.rs | 64 +- crates/openshell-driver-podman/README.md | 11 +- crates/openshell-driver-podman/src/client.rs | 66 +- crates/openshell-driver-podman/src/config.rs | 10 + .../openshell-driver-podman/src/container.rs | 46 +- crates/openshell-driver-podman/src/driver.rs | 342 ++++++++- crates/openshell-driver-podman/src/main.rs | 9 + crates/openshell-driver-vm/README.md | 7 + crates/openshell-driver-vm/src/driver.rs | 94 ++- crates/openshell-driver-vm/src/main.rs | 9 + crates/openshell-gateway/Cargo.toml | 1 + crates/openshell-gateway/src/vm.rs | 13 + .../src/compute/driver_config.rs | 55 ++ crates/openshell-server/src/compute/mod.rs | 201 +++++- crates/openshell-server/src/grpc/sandbox.rs | 4 + crates/openshell-server/src/lib.rs | 5 + crates/openshell-server/src/test_support.rs | 7 + .../openshell-workspace/templates/role.yaml | 4 + deploy/helm/openshell/README.md | 3 + .../helm/openshell/templates/clusterrole.yaml | 11 + .../openshell/templates/gateway-config.yaml | 10 + deploy/helm/openshell/templates/role.yaml | 4 + .../openshell/tests/gateway_config_test.yaml | 43 ++ deploy/helm/openshell/values.yaml | 7 + docs/kubernetes/sandbox-runtime.mdx | 16 + docs/reference/gateway-config.mdx | 123 ++++ docs/reference/sandbox-compute-drivers.mdx | 39 +- e2e/parity/test.sh | 6 + e2e/support/podman-gateway-config.sh | 3 + e2e/with-docker-gateway.sh | 3 + e2e/with-kube-gateway.sh | 3 + proto/compute_driver.proto | 4 + skills/debug-openshell-cluster/SKILL.md | 16 + skills/openshell-cli/SKILL.md | 11 +- 51 files changed, 3231 insertions(+), 82 deletions(-) create mode 100644 crates/openshell-core/src/resource_admission.rs create mode 100644 crates/openshell-driver-kubernetes/src/resource_admission.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 077f70ccd8..2df971436f 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -69,6 +69,13 @@ mise run helm:skaffold:dev mise run helm:skaffold:run ``` +Resource admission defaults to enabled and caller driver config to disabled. +Driver-config scenarios need an explicit `allowDriverConfig` opt-in; external +attachments also need administrator-controlled approval labels in the target +namespace. GPU attachments are exempt from labels. Managed workspace image-pull +Secrets are copied from an approved source in the gateway namespace; do not grant +approval to the gateway database PVC or disable admission to make tests pass. + The Skaffold flow builds distinct `gateway`, `sandbox`, and `supervisor` images and deploys the OpenShell Helm chart. The Kubernetes driver creates a capability-free workload Pod and a directly managed capability-free supervisor diff --git a/Cargo.lock b/Cargo.lock index 7c16585909..53a3654ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4382,6 +4382,7 @@ dependencies = [ "openshell-server", "rustix 1.1.4", "serde", + "serde_json", "tempfile", "tokio", "toml", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 1ed9f9d900..44415009c8 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -16,6 +16,22 @@ the common protocol owns process, identity, TCP, DNS, and forwarding semantics. ## Driver Contract +External resource admission is an operator-owned boundary shared by drivers. +The gateway gates caller driver JSON independently from attachment approval. +Drivers resolve the complete effective attachment inventory against authoritative +resource labels before launch and on reuse. Missing labels or an unsupported +resolver deny access; GPU attachments are an explicit temporary exception. +Fresh sandbox-private resources instead require verified provisioning ownership. +Workload metadata must not grant approval or override admission evidence. + +The shared evaluator lives in `openshell-core`; native resolution remains in +each driver. External drivers acknowledge the effective versioned policy through +capabilities, and policy mismatch prevents activation or new launch operations. +Trusted deployment configuration can explicitly disable label admission, but +that opt-out does not waive other ownership and isolation checks. This boundary +assumes operators control approval metadata and runtime resource replacement; +it does not provide atomic mount authorization or instantaneous revocation. + Each runtime receives a sandbox spec and canonical policy from the gateway and is responsible for: @@ -449,11 +465,14 @@ management. RBAC uses a namespace-scoped Role. Each new namespace receives a ServiceAccount and the configured gateway-only SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the driver's source namespace on every sandbox create so registry credential -rotations propagate. The namespace also copies OpenShift SCC UID-range and -supplemental-group annotations from the gateway namespace when present. The -driver deletes the namespace during workspace deletion. The workspace remains -durably `Terminating` until the Kubernetes API accepts namespace cleanup, so a -transient failure can be retried. Namespace deletion uses the fetched UID as a +rotations propagate. Resource admission first validates the source as shared +operator infrastructure, and copies carry gateway and workspace ownership +labels; an unrelated existing target is never adopted. The namespace also copies +OpenShift SCC UID-range and supplemental-group annotations from the gateway +namespace when present. The driver deletes the namespace during workspace +deletion. The workspace remains durably `Terminating` until the Kubernetes API +accepts namespace cleanup, so a transient failure can be retried. Namespace +deletion uses the fetched UID as a precondition to avoid deleting a replacement namespace. Requires a non-empty `gateway_id` (validated as a DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-character diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 778cc7dfd1..06ca33c9dc 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1465,8 +1465,9 @@ enum SandboxCommands { #[arg(long)] memory: Option, - /// Experimental driver-keyed JSON object for driver-specific sandbox settings. - /// Validation behavior is not yet finalized. + /// Driver-keyed JSON object for driver-specific sandbox settings. + /// Disabled unless the gateway administrator enables `allow_driver_config`. + /// External resource attachments still require approval labels. /// /// For Kubernetes, pass a value such as /// `{"kubernetes":{"pod":{"node_selector":{"pool":"gpu"}}}}`. @@ -1885,7 +1886,8 @@ enum SandboxTemplateCommands { #[arg(long, num_args = 0..=1, value_name = "COUNT", default_missing_value = "", value_parser = parse_gpu_request)] gpu: Option, - /// Experimental driver-keyed JSON object for driver-specific sandbox settings. + /// Driver-keyed JSON object for driver-specific sandbox settings. + /// Requires administrator opt-in; resource admission still applies. #[arg(long, value_name = "JSON")] driver_config_json: Option, diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index b310aba8ac..472175f511 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -45,6 +45,7 @@ pub mod proposals; pub mod proto; pub mod proto_struct; pub mod provider_credentials; +pub mod resource_admission; pub mod rpc_error; pub mod sandbox_env; pub mod sandbox_generation; diff --git a/crates/openshell-core/src/resource_admission.rs b/crates/openshell-core/src/resource_admission.rs new file mode 100644 index 0000000000..7d20c405f4 --- /dev/null +++ b/crates/openshell-core/src/resource_admission.rs @@ -0,0 +1,415 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Operator-owned external resource admission. GPU attachments are temporarily +//! exempt; this policy must not be interpreted as permission to attach host paths. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::driver_utils::{LABEL_GATEWAY_ID, LABEL_MANAGED_BY, LABEL_SANDBOX_WORKSPACE}; + +const WORKSPACE_PLACEHOLDER: &str = "${workspace}"; +const RESERVED_LABEL_KEYS: [&str; 3] = + [LABEL_MANAGED_BY, LABEL_GATEWAY_ID, LABEL_SANDBOX_WORKSPACE]; + +/// Label policy shared by every compute driver. A supplied map replaces defaults. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ResourceAdmissionConfig { + pub enabled: bool, + pub required_labels: BTreeMap, +} + +impl Default for ResourceAdmissionConfig { + fn default() -> Self { + Self { + enabled: true, + required_labels: BTreeMap::from([ + ("openshell.ai/sandbox-attachable".into(), "true".into()), + ( + "openshell.ai/sandbox-attachable-workspace".into(), + WORKSPACE_PLACEHOLDER.into(), + ), + ]), + } + } +} + +/// Effective policy acknowledged by a driver, including the independent JSON gate. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct DriverAdmissionConfig { + pub allow_driver_config: bool, + pub resource_admission: ResourceAdmissionConfig, +} + +impl DriverAdmissionConfig { + pub fn validate(&self) -> Result<(), String> { + self.resource_admission.validate() + } + + /// Versioned acknowledgement. Compare parsed policies, never an untrusted flag. + #[must_use] + pub fn acknowledgement(&self) -> String { + // These types contain only strings, booleans and string-keyed maps. + format!( + "v1:{}", + serde_json::to_string(self).expect("serializable admission policy") + ) + } + + pub fn verify_acknowledgement(&self, value: &str) -> Result<(), String> { + self.validate()?; + if !self.resource_admission.enabled && value.is_empty() { + // Explicit legacy-driver opt-out. The gateway still gates caller JSON. + return Ok(()); + } + let policy: Self = value + .strip_prefix("v1:") + .ok_or("compute driver does not acknowledge resource admission v1") + .and_then(|json| { + serde_json::from_str(json).map_err(|_| "invalid driver admission acknowledgement") + })?; + if policy != *self { + return Err( + "compute driver admission policy differs from gateway configuration".into(), + ); + } + Ok(()) + } +} + +impl std::str::FromStr for DriverAdmissionConfig { + type Err = String; + fn from_str(value: &str) -> Result { + let policy: Self = serde_json::from_str(value).map_err(|error| error.to_string())?; + policy.validate()?; + Ok(policy) + } +} + +/// Reserved driver-owned runtime metadata; caller labels must never override it. +pub const CONFIG_USED_LABEL: &str = "openshell.ai/caller-driver-config-used"; +pub const IDENTITIES_LABEL: &str = "openshell.ai/resource-admission-identities"; + +pub fn check_config_provenance(allowed: bool, recorded: Option<&str>) -> Result<(), tonic::Status> { + match recorded { + Some("false") => Ok(()), + Some("true") if allowed => Ok(()), + _ => Err(tonic::Status::failed_precondition( + "sandbox uses forbidden driver config or lacks admission provenance; recreate it", + )), + } +} + +/// Check caller config before resource lookups, including direct driver RPCs. +pub fn check_driver_config( + allowed: bool, + config: Option<&prost_types::Struct>, +) -> Result<(), tonic::Status> { + if !allowed && config.is_some_and(|config| !config.fields.is_empty()) { + return Err(tonic::Status::failed_precondition( + "caller driver config is disabled; a gateway administrator must enable allow_driver_config", + )); + } + Ok(()) +} + +pub fn check_sandbox_driver_config( + allowed: bool, + sandbox: &crate::proto::compute::v1::DriverSandbox, +) -> Result<(), tonic::Status> { + check_driver_config( + allowed, + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .and_then(|template| template.driver_config.as_ref()), + ) +} + +impl ResourceAdmissionConfig { + pub fn validate(&self) -> Result<(), String> { + if self.enabled && self.required_labels.is_empty() { + return Err( + "resource_admission.required_labels must not be empty while enabled".into(), + ); + } + for (key, value) in &self.required_labels { + if RESERVED_LABEL_KEYS.contains(&key.as_str()) { + return Err(format!( + "resource admission label key is reserved for driver-owned metadata: {key}" + )); + } + if !valid_label_key(key) { + return Err(format!("invalid resource admission label key: {key}")); + } + if value != WORKSPACE_PLACEHOLDER && !valid_label_value(value) { + return Err(format!( + "invalid resource admission label value for {key}; only whole-value ${{workspace}} substitution is supported" + )); + } + } + if self.enabled + && self + .required_labels + .values() + .all(|value| value == WORKSPACE_PLACEHOLDER) + { + return Err( + "resource_admission.required_labels must include a shared approval label".into(), + ); + } + Ok(()) + } + + /// Evaluate labels read from the resource authority, never caller metadata. + pub fn admit<'a>( + &self, + workspace: &str, + labels: impl IntoIterator, + ) -> Result<(), tonic::Status> { + self.admit_labels(Some(workspace), labels) + } + + /// Evaluate a shared operator resource. Fixed approval labels still apply, + /// but workspace placeholders do not: cluster-scoped and intentionally + /// shared resources cannot carry one tenant's identity. + pub fn admit_shared<'a>( + &self, + labels: impl IntoIterator, + ) -> Result<(), tonic::Status> { + self.admit_labels(None, labels) + } + + fn admit_labels<'a>( + &self, + workspace: Option<&str>, + labels: impl IntoIterator, + ) -> Result<(), tonic::Status> { + self.validate() + .map_err(tonic::Status::failed_precondition)?; + if !self.enabled { + return Ok(()); + } + let labels: BTreeMap<_, _> = labels.into_iter().collect(); + for (key, expected) in &self.required_labels { + let expected = if expected == WORKSPACE_PLACEHOLDER { + let Some(workspace) = workspace else { + continue; + }; + if workspace.is_empty() || !valid_label_value(workspace) { + return Err(tonic::Status::failed_precondition( + "resource not admitted: invalid workspace identity", + )); + } + workspace + } else { + expected + }; + if labels.get(key).map(|value| value.as_str()) != Some(expected) { + return Err(tonic::Status::failed_precondition( + "external resource not admitted by required labels", + )); + } + } + Ok(()) + } + + pub fn reject_unlabelable(&self, kind: &str) -> Result<(), tonic::Status> { + if self.enabled { + Err(tonic::Status::failed_precondition(format!( + "{kind} cannot be attached while resource admission is enabled: no trusted label resolver" + ))) + } else { + Ok(()) + } + } +} + +fn valid_label_value(value: &str) -> bool { + value.is_empty() + || (value.len() <= 63 + && value + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && value + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) + && value + .bytes() + .all(|c| c.is_ascii_alphanumeric() || b"-_.".contains(&c))) +} + +fn valid_label_key(key: &str) -> bool { + let (prefix, name) = key + .split_once('/') + .map_or((None, key), |(prefix, name)| (Some(prefix), name)); + !name.is_empty() + && valid_label_value(name) + && prefix.is_none_or(|prefix| { + !prefix.is_empty() + && prefix.len() <= 253 + && prefix.split('.').all(|part| { + !part.is_empty() + && part.len() <= 63 + && part + .bytes() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'-') + && part + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && part + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn labels(workspace: &str) -> BTreeMap { + BTreeMap::from([ + ("openshell.ai/sandbox-attachable".into(), "true".into()), + ( + "openshell.ai/sandbox-attachable-workspace".into(), + workspace.into(), + ), + ]) + } + + #[test] + fn rejects_unapproved_and_cross_workspace_resources() { + let policy = ResourceAdmissionConfig::default(); + assert!(policy.admit("a", &labels("a")).is_ok()); + assert!(policy.admit("b", &labels("a")).is_err()); + assert!(policy.admit_shared(&labels("a")).is_ok()); + assert!(policy.admit("a", &BTreeMap::new()).is_err()); + assert!(policy.admit_shared(&BTreeMap::new()).is_err()); + assert!(policy.admit("", &labels("")).is_err()); + } + + #[test] + fn replacement_map_and_explicit_empty_policy() { + let policy: ResourceAdmissionConfig = + serde_json::from_str(r#"{"required_labels":{"example.com/approved":"yes"}}"#).unwrap(); + assert_eq!(policy.required_labels.len(), 1); + assert!( + policy + .admit( + "a", + &BTreeMap::from([("example.com/approved".into(), "yes".into())]) + ) + .is_ok() + ); + let empty: ResourceAdmissionConfig = + serde_json::from_str(r#"{"required_labels":{}}"#).unwrap(); + assert!(empty.validate().is_err()); + let workspace_only: ResourceAdmissionConfig = serde_json::from_str( + r#"{"required_labels":{"openshell.ai/sandbox-attachable-workspace":"${workspace}"}}"#, + ) + .unwrap(); + assert!(workspace_only.validate().is_err()); + assert!( + ResourceAdmissionConfig { + enabled: false, + ..empty + } + .validate() + .is_ok() + ); + } + + #[test] + fn rejects_driver_owned_label_keys() { + assert!(ResourceAdmissionConfig::default().validate().is_ok()); + for key in RESERVED_LABEL_KEYS { + let policy = ResourceAdmissionConfig { + required_labels: BTreeMap::from([ + ("example.com/approved".into(), "true".into()), + (key.into(), "operator-value".into()), + ]), + ..Default::default() + }; + assert_eq!( + policy.validate(), + Err(format!( + "resource admission label key is reserved for driver-owned metadata: {key}" + )) + ); + } + } + + #[test] + fn rejects_invalid_labels_and_interpolation() { + for value in [ + "${workspace.id}", + "prefix-${workspace}", + "$HOME", + "*", + "x/y", + ] { + let policy = ResourceAdmissionConfig { + required_labels: BTreeMap::from([("approved".into(), value.into())]), + ..Default::default() + }; + assert!(policy.validate().is_err(), "{value}"); + } + for key in ["", "/a", "bad_domain/a", "a/b/c", "-a", "a/"] { + assert!(!valid_label_key(key), "{key}"); + } + } + + #[test] + fn json_gate_is_independent_of_label_opt_out() { + let config = prost_types::Struct { + fields: BTreeMap::from([("gpu_device_ids".into(), prost_types::Value::default())]), + }; + for enabled in [false, true] { + let policy = DriverAdmissionConfig { + resource_admission: ResourceAdmissionConfig { + enabled, + ..Default::default() + }, + ..Default::default() + }; + assert!(check_driver_config(policy.allow_driver_config, Some(&config)).is_err()); + assert!(check_driver_config(true, Some(&config)).is_ok()); + assert!(check_driver_config(false, Some(&prost_types::Struct::default())).is_ok()); + assert!(check_driver_config(false, None).is_ok()); + } + } + + #[test] + fn driver_acknowledgement_must_match_effective_policy() { + let policy = DriverAdmissionConfig::default(); + assert!( + policy + .verify_acknowledgement(&policy.acknowledgement()) + .is_ok() + ); + assert!(policy.verify_acknowledgement("").is_err()); + let unsafe_policy = DriverAdmissionConfig { + resource_admission: ResourceAdmissionConfig { + enabled: false, + ..Default::default() + }, + ..Default::default() + }; + assert!( + policy + .verify_acknowledgement(&unsafe_policy.acknowledgement()) + .is_err() + ); + assert!(unsafe_policy.verify_acknowledgement("").is_ok()); + } +} diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index aa45b14745..096181a69f 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -15,6 +15,12 @@ metadata, and flushes spans during graceful shutdown. ## Runtime Model +Caller driver config is disabled by default. Existing volumes require +administrator-controlled approval labels; raw bind mounts have no supported +label resolver and are denied under enforcement. GPU devices are temporarily +exempt. Inspect labels again before launch, restart, and during reconciliation. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). + The driver creates two containers for each sandbox: - `openshell-sandbox` is PID 1 in the workload container. It owns the workload @@ -98,7 +104,8 @@ The gateway forwards the `docker` block from `--driver-config-json`. Supported mount types are: - `bind`: an absolute daemon-host path, allowed only when - `[openshell.drivers.docker].enable_bind_mounts = true`. + `[openshell.drivers.docker].enable_bind_mounts = true` and label admission + is explicitly disabled. - `volume`: an existing named volume. The driver never creates or removes a user-supplied volume. Bind-backed local volumes require `enable_bind_mounts = true`. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 9bef89f353..c76c4c838b 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -154,6 +154,10 @@ fn provisioning_span( #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] pub struct DockerComputeConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// Docker API Unix socket. When unset, use the socket selected by gateway /// auto-detection, falling back to `/var/run/docker.sock` for an explicitly /// configured Docker driver. @@ -225,6 +229,7 @@ pub struct DockerComputeConfig { impl DockerComputeConfig { /// Validate startup configuration without connecting to Docker. pub fn validate_configuration(&self, gateway_bind_address: SocketAddr) -> CoreResult<()> { + self.resource_admission.validate().map_err(Error::config)?; if let Some(socket_path) = self.socket_path.as_deref() && socket_path.to_str().is_none() { @@ -254,6 +259,9 @@ impl Default for DockerComputeConfig { fn default() -> Self { Self { socket_path: None, + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), default_image: openshell_core::image::default_sandbox_image(), image_pull_policy: ImagePullPolicy::default(), sandbox_label: "default".to_string(), @@ -283,6 +291,8 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { + allow_driver_config: bool, + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, default_image: String, image_pull_policy: ImagePullPolicy, sandbox_namespace: String, @@ -827,6 +837,10 @@ impl DockerComputeDriver { gateway_log_level: &str, docker_config: &DockerComputeConfig, ) -> CoreResult { + docker_config + .resource_admission + .validate() + .map_err(Error::config)?; let socket_path = docker_config .socket_path .clone() @@ -930,6 +944,8 @@ impl DockerComputeDriver { gpu, sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, + allow_driver_config: docker_config.allow_driver_config, + resource_admission: docker_config.resource_admission.clone(), upstream_proxy: docker_config.upstream_proxy.clone(), provider_spiffe_workload_api_socket: docker_config .provider_spiffe_workload_api_socket @@ -966,6 +982,11 @@ impl DockerComputeDriver { fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: "docker".to_string(), driver_version: openshell_core::VERSION.to_string(), default_image: self.config.default_image.clone(), @@ -1008,6 +1029,10 @@ impl DockerComputeDriver { sandbox: &'a DriverSandbox, config: &DockerDriverRuntimeConfig, ) -> Result, Status> { + openshell_core::resource_admission::check_sandbox_driver_config( + config.allow_driver_config, + sandbox, + )?; let spec = sandbox .spec .as_ref() @@ -1022,6 +1047,16 @@ impl DockerComputeDriver { let driver_config = DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?; + for mount in &driver_config.mounts { + if matches!( + mount, + DockerDriverMountConfig::Bind { .. } | DockerDriverMountConfig::Image { .. } + ) { + config + .resource_admission + .reject_unlabelable("host bind or image mount")?; + } + } let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref()); Self::validate_gpu_request(gpu_requirements, config.gpu.cdi_supported, &driver_config)?; Ok(ValidatedDockerSandbox { @@ -1096,11 +1131,23 @@ impl DockerComputeDriver { async fn validate_user_volume_mounts_available( &self, driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { + workspace: &str, + ) -> Result, Status> { + let mut identities = std::collections::BTreeMap::new(); for mount in &driver_config.mounts { if let DockerDriverMountConfig::Volume { source, .. } = mount { match self.docker.inspect_volume(source).await { Ok(volume) => { + identities.insert(source.clone(), docker_volume_identity(&volume)); + self.config + .resource_admission + .admit(workspace, &volume.labels) + .map_err(|error| { + Status::new( + error.code(), + format!("docker volume '{source}': {}", error.message()), + ) + })?; if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) { return Err(Status::failed_precondition(format!( @@ -1119,7 +1166,7 @@ impl DockerComputeDriver { } } } - Ok(()) + Ok(identities) } async fn refresh_gpu_inventory(&self) -> Result<(), Status> { @@ -1135,6 +1182,114 @@ impl DockerComputeDriver { Ok(()) } + async fn admit_container_resources( + &self, + container: &bollard::models::ContainerInspectResponse, + ) -> Result<(), Status> { + if self.config.allow_driver_config && !self.config.resource_admission.enabled { + return Ok(()); + } + let labels = container + .config + .as_ref() + .and_then(|config| config.labels.as_ref()) + .ok_or_else(|| Status::failed_precondition("sandbox lacks admission metadata"))?; + openshell_core::resource_admission::check_config_provenance( + self.config.allow_driver_config, + labels + .get(openshell_core::resource_admission::CONFIG_USED_LABEL) + .map(String::as_str), + )?; + if !self.config.resource_admission.enabled { + return Ok(()); + } + let workspace = labels + .get(LABEL_SANDBOX_WORKSPACE) + .ok_or_else(|| Status::failed_precondition("sandbox lacks workspace"))?; + let sandbox_id = labels + .get(LABEL_SANDBOX_ID) + .ok_or_else(|| Status::failed_precondition("sandbox lacks identity"))?; + let expected: std::collections::BTreeMap = labels + .get(openshell_core::resource_admission::IDENTITIES_LABEL) + .and_then(|value| serde_json::from_str(value).ok()) + .ok_or_else(|| Status::failed_precondition("sandbox lacks resource identity record"))?; + let mut actual = std::collections::BTreeMap::new(); + let anonymous_targets: Vec = labels + .get("openshell.ai/private-image-volume-targets") + .and_then(|value| serde_json::from_str(value).ok()) + .unwrap_or_default(); + for mount in container.mounts.as_deref().unwrap_or_default() { + match mount.typ { + Some(bollard::models::MountPointTypeEnum::VOLUME) => { + let name = mount + .name + .as_deref() + .ok_or_else(|| Status::failed_precondition("volume lacks identity"))?; + let volume = self.docker.inspect_volume(name).await.map_err(|error| { + if is_not_found_error(&error) { + Status::failed_precondition("attached volume no longer exists") + } else { + Status::unavailable("volume admission lookup failed") + } + })?; + if name == docker_channel_volume_name_by_id(sandbox_id, &self.config) { + if volume.driver != "local" + || !volume.options.is_empty() + || volume.labels.get(LABEL_SANDBOX_ID) != Some(sandbox_id) + || volume.labels.get(LABEL_MANAGED_BY).map(String::as_str) + != Some(LABEL_MANAGED_BY_VALUE) + { + return Err(Status::failed_precondition( + "sandbox-private volume ownership changed", + )); + } + } else if !expected.contains_key(name) + && mount + .destination + .as_ref() + .is_some_and(|destination| anonymous_targets.contains(destination)) + { + // The immutable create spec requested a fresh anonymous + // image volume at this target, never an existing name. + if volume.driver != "local" || !volume.options.is_empty() { + return Err(Status::failed_precondition( + "image-private volume backing changed", + )); + } + } else { + actual.insert(name.to_string(), docker_volume_identity(&volume)); + self.config + .resource_admission + .admit(workspace, &volume.labels) + .map_err(|error| { + Status::new( + error.code(), + format!("docker volume '{name}': {}", error.message()), + ) + })?; + if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) + { + return Err(Status::failed_precondition( + "bind-backed volume is disabled", + )); + } + } + } + Some(bollard::models::MountPointTypeEnum::TMPFS) => {} + _ => self + .config + .resource_admission + .reject_unlabelable("effective container mount")?, + } + } + if actual != expected { + return Err(Status::failed_precondition( + "external volume identity or attachment inventory changed", + )); + } + Ok(()) + } + async fn resolve_gpu_cdi_devices( &self, gpu_requirements: Option<&GpuResourceRequirements>, @@ -1273,7 +1428,13 @@ impl DockerComputeDriver { } } Ok(inspected) if summary.state == Some(ContainerSummaryStateEnum::RUNNING) => { - if let Err(status) = validate_docker_outer_fence(&inspected) { + let admission = match validate_docker_outer_fence(&inspected) { + Ok(()) => self.admit_container_resources(&inspected).await, + Err(error) => Err(error), + }; + if let Err(status) = admission + && status.code() == tonic::Code::FailedPrecondition + { let context = self .control_failure_context(sandbox.clone(), container_id.to_string()); handle_docker_runtime_failure( @@ -1313,7 +1474,7 @@ impl DockerComputeDriver { async fn create_sandbox_inner(&self, sandbox: &DriverSandbox) -> Result<(), Status> { let validated = Self::validated_sandbox(sandbox, &self.config)?; Self::validate_sandbox_auth(sandbox)?; - self.validate_user_volume_mounts_available(&validated.driver_config) + self.validate_user_volume_mounts_available(&validated.driver_config, &sandbox.workspace) .await?; let _ = self .resolve_gpu_cdi_devices( @@ -1472,7 +1633,7 @@ impl DockerComputeDriver { )); } }; - let create_body = match build_container_create_body_for_image( + let mut create_body = match build_container_create_body_for_image( sandbox, &self.config, &validated.driver_config, @@ -1491,6 +1652,27 @@ impl DockerComputeDriver { )); } }; + let identities = match self + .validate_user_volume_mounts_available(&validated.driver_config, &sandbox.workspace) + .await + { + Ok(identities) => identities, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::from_admission_status(status)); + } + }; + create_body + .labels + .get_or_insert_with(Default::default) + .insert( + openshell_core::resource_admission::IDENTITIES_LABEL.into(), + serde_json::to_string(&identities).map_err(|error| { + DockerProvisioningFailure::new("ResourceAdmissionDenied", error.to_string()) + })?, + ); let create_result = async { openshell_otel::record_error_result( self.docker @@ -1544,7 +1726,10 @@ impl DockerComputeDriver { )); } }; - let outer_fence_error = validate_docker_outer_fence(&inspected).err(); + let outer_fence_error = match validate_docker_outer_fence(&inspected) { + Ok(()) => self.admit_container_resources(&inspected).await.err(), + Err(error) => Some(error), + }; drop(inspected); if let Some(status) = outer_fence_error { let _ = self @@ -2224,6 +2409,7 @@ impl DockerComputeDriver { .await .map_err(|error| internal_status("inspect Docker sandbox outer fence", error))?; validate_docker_outer_fence(&inspected)?; + self.admit_container_resources(&inspected).await?; let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); let previous_finished_at = if state == ContainerSummaryStateEnum::EXITED { inspected @@ -3037,7 +3223,7 @@ impl ComputeDriver for DockerComputeDriver { .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; let validated = Self::validated_sandbox(&sandbox, &self.config)?; - self.validate_user_volume_mounts_available(&validated.driver_config) + self.validate_user_volume_mounts_available(&validated.driver_config, &sandbox.workspace) .await?; let _ = self .resolve_gpu_cdi_devices( @@ -3272,6 +3458,15 @@ impl DockerProvisioningFailure { fn from_status(reason: &'static str, status: Status) -> Self { Self::new(reason, status.message()) } + + fn from_admission_status(status: Status) -> Self { + let reason = if status.code() == tonic::Code::FailedPrecondition { + "ResourceAdmissionDenied" + } else { + "ResourceAdmissionLookupFailed" + }; + Self::from_status(reason, status) + } } fn sandbox_image(sandbox: &DriverSandbox) -> Option { @@ -3779,6 +3974,11 @@ fn docker_tmpfs_option(option: &str) -> Result, Status> { } } +fn docker_volume_identity(volume: &bollard::models::Volume) -> serde_json::Value { + serde_json::json!({"name": volume.name, "driver": volume.driver, "options": volume.options, + "created_at": volume.created_at, "mountpoint": volume.mountpoint}) +} + fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { volume.driver == "local" && volume.options.get("o").is_some_and(|options| { @@ -5503,6 +5703,19 @@ fn build_container_create_body_for_image( }] }); let mut labels = template.labels.clone(); + labels.insert( + "openshell.ai/private-image-volume-targets".into(), + serde_json::to_string(&image.volumes) + .map_err(|error| Status::internal(error.to_string()))?, + ); + labels.insert( + openshell_core::resource_admission::CONFIG_USED_LABEL.into(), + template + .driver_config + .as_ref() + .is_some_and(|config| !config.fields.is_empty()) + .to_string(), + ); labels.insert( LABEL_MANAGED_BY.to_string(), LABEL_MANAGED_BY_VALUE.to_string(), diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index 8630d37c11..746c62981c 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -14,6 +14,9 @@ use tracing::info; #[derive(Debug, Parser)] #[command(name = "openshell-driver-docker", version = VERSION)] struct Args { + /// Override the operator admission policy in the driver TOML file. + #[arg(long, env = "OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON")] + admission_config_json: Option, /// Public compute-driver Unix socket used by the gateway. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] bind_socket: PathBuf, @@ -54,7 +57,12 @@ async fn main() -> Result<()> { ); let config_source = std::fs::read_to_string(&args.config).into_diagnostic()?; - let docker_config: DockerComputeConfig = toml::from_str(&config_source).into_diagnostic()?; + let mut docker_config: DockerComputeConfig = + toml::from_str(&config_source).into_diagnostic()?; + if let Some(policy) = args.admission_config_json { + docker_config.allow_driver_config = policy.allow_driver_config; + docker_config.resource_admission = policy.resource_admission; + } let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config) .await .into_diagnostic()?; diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 0908fc2739..aa97d0ff09 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -79,6 +79,58 @@ fn test_sandbox() -> DriverSandbox { } } +#[test] +fn admission_defaults_reject_even_gpu_driver_json_but_not_public_gpu_requests() { + let mut config = runtime_config(); + config.allow_driver_config = false; + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + config.gpu.cdi_supported = true; + let mut sandbox = test_sandbox(); + let spec = sandbox.spec.as_mut().unwrap(); + spec.resource_requirements = Some(gpu_resources(Some(1))); + assert!(DockerComputeDriver::validate_sandbox(&sandbox, &config).is_ok()); + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(cdi_devices_config(&["nvidia.com/gpu=0"])); + let error = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + config.allow_driver_config = true; + assert!(DockerComputeDriver::validate_sandbox(&sandbox, &config).is_ok()); +} + +#[test] +fn admission_blocks_unlabelable_bind_mount_even_when_bind_mounts_enabled() { + let mut config = runtime_config(); + config.enable_bind_mounts = true; + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + let mut sandbox = test_sandbox(); + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some( + openshell_core::proto_struct::json_object_to_struct( + serde_json::json!({"mounts":[{"type":"bind","source":"/srv/data","target":"/data"}]}) + .as_object() + .unwrap() + .clone(), + ) + .unwrap(), + ); + let error = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); + assert!(error.message().contains("no trusted label resolver")); +} + fn cdi_devices_config(device_ids: &[&str]) -> prost_types::Struct { list_string_driver_config("cdi_devices", device_ids) } @@ -118,6 +170,12 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { + // Existing lifecycle fixtures exercise the explicitly opted-out contract. + allow_driver_config: true, + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig { + enabled: false, + ..Default::default() + }, default_image: "image:latest".to_string(), image_pull_policy: ImagePullPolicy::IfNotPresent, sandbox_namespace: "default".to_string(), @@ -237,6 +295,76 @@ async fn capabilities_reject_missing_gateway_metadata() { ); } +#[tokio::test] +#[ignore = "requires a local Docker daemon; creates and removes only isolated test volumes"] +async fn live_docker_resource_admission_checks_native_volume_labels() { + let temporary = TempDir::new().unwrap(); + let suffix = temporary.path().file_name().unwrap().to_str().unwrap(); + let mut config = runtime_config(); + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + let mut driver = test_driver_with_config(config); + driver.docker = Arc::new(Docker::connect_with_local_defaults().unwrap()); + for (index, (workspace, approved)) in [ + (None, false), + (Some("other"), false), + (Some("team-a"), true), + ] + .into_iter() + .enumerate() + { + let name = format!("openshell-admission-{suffix}-{index}"); + assert!(driver.docker.inspect_volume(&name).await.is_err()); + let labels = workspace.map(|workspace| { + HashMap::from([ + ("openshell.ai/sandbox-attachable".into(), "true".into()), + ( + "openshell.ai/sandbox-attachable-workspace".into(), + workspace.into(), + ), + ]) + }); + driver + .docker + .create_volume(VolumeCreateRequest { + name: Some(name.clone()), + labels, + ..Default::default() + }) + .await + .unwrap(); + let mut results = Vec::new(); + for read_only in [true, false] { + let mounts = serde_json::from_value(serde_json::json!({"mounts":[{ + "type":"volume", "source":name, "target":"/external", "read_only":read_only + }]})) + .unwrap(); + let result = driver + .validate_user_volume_mounts_available(&mounts, "team-a") + .await; + if !approved { + assert!( + result + .as_ref() + .unwrap_err() + .message() + .contains(&format!("docker volume '{name}'")) + ); + } + results.push(result.is_ok()); + } + driver + .docker + .remove_volume( + &name, + None::, + ) + .await + .unwrap(); + assert_eq!(results, vec![approved; 2]); + } +} + type TestDriverClient = openshell_core::proto::compute::v1::compute_driver_client::ComputeDriverClient< tonic::transport::Channel, @@ -3351,3 +3479,18 @@ async fn different_start_generation_is_rejected() { assert_eq!(error.code(), tonic::Code::FailedPrecondition); assert!(error.message().contains("generation-one")); } + +#[test] +fn admission_provisioning_failure_distinguishes_denials_from_lookup_failures() { + let denied = DockerProvisioningFailure::from_admission_status(Status::failed_precondition( + "volume is not admitted", + )); + assert_eq!(denied.reason, "ResourceAdmissionDenied"); + assert_eq!(denied.message, "volume is not admitted"); + + let lookup = DockerProvisioningFailure::from_admission_status(Status::internal( + "inspect docker volume failed", + )); + assert_eq!(lookup.reason, "ResourceAdmissionLookupFailed"); + assert_eq!(lookup.message, "inspect docker volume failed"); +} diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 16c12d0505..6e1578cee4 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -2,6 +2,13 @@ Kubernetes-backed compute driver for OpenShell cluster deployments. +Caller driver config is disabled by default. External resource references need +administrator-controlled approval labels in every workspace mode, including +before restart and scheduling-gate release. GPU devices are temporarily exempt. +Managed workspace image-pull Secrets are copied only after the configured source +Secret passes shared-resource admission; copies carry gateway ownership metadata. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). + The driver uses the Kubernetes API to create, delete, fetch, and watch sandbox custom resources. It runs in-process with the gateway server and supports three workspace namespace modes via `workspace_mode`: diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 1574bee36f..087c7a5a41 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -166,6 +166,10 @@ where #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesComputeConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// How workspaces map to Kubernetes namespaces. `"shared"` (default) /// renders all sandboxes into `namespace`; `"managed"` creates per-workspace /// namespaces on demand; `"operator"` uses pre-provisioned namespaces. @@ -326,6 +330,9 @@ impl Default for KubernetesComputeConfig { fn default() -> Self { Self { workspace_mode: WorkspaceMode::default(), + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), gateway_id: DEFAULT_GATEWAY_ID.to_string(), namespace: DEFAULT_K8S_NAMESPACE.to_string(), operator_namespace_label: None, diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f559a4d0d1..356fdc249d 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -267,6 +267,17 @@ impl From for openshell_core::ComputeDriverError { /// This prevents gRPC handlers from blocking indefinitely when the k8s /// API server is unreachable or slow. const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30); +fn admission_error(error: tonic::Status) -> KubernetesDriverError { + match error.code() { + tonic::Code::InvalidArgument => { + KubernetesDriverError::InvalidArgument(error.message().into()) + } + tonic::Code::FailedPrecondition => { + KubernetesDriverError::Precondition(error.message().into()) + } + _ => KubernetesDriverError::Message(error.message().into()), + } +} const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); /// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5); @@ -712,6 +723,10 @@ impl KubernetesComputeDriver { config: KubernetesComputeConfig, shutdown_rx: tokio::sync::watch::Receiver, ) -> Result { + config + .resource_admission + .validate() + .map_err(KubernetesDriverError::Precondition)?; config .validate_workspace_mode() .map_err(KubernetesDriverError::Precondition)?; @@ -793,6 +808,11 @@ impl KubernetesComputeDriver { pub fn capabilities(&self) -> Result { Ok(GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: "kubernetes".to_string(), driver_version: openshell_core::VERSION.to_string(), default_image: self.config.default_image.clone(), @@ -1261,6 +1281,7 @@ impl KubernetesComputeDriver { async fn ensure_image_pull_secrets( &self, namespace: &str, + workspace: &str, ) -> Result<(), KubernetesDriverError> { let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); let target_api: Api = Api::namespaced(self.client.clone(), namespace); @@ -1285,7 +1306,55 @@ impl KubernetesComputeDriver { } }; - let copy = image_pull_secret_copy(secret_name, namespace, source); + self.config + .resource_admission + .admit_shared( + source + .metadata + .labels + .as_ref() + .into_iter() + .flat_map(|labels| labels.iter()), + ) + .map_err(|error| { + admission_error(tonic::Status::new( + error.code(), + format!( + "Secret '{}/{}': {}", + self.config.namespace, + secret_name, + error.message() + ), + )) + })?; + + let existing = tokio::time::timeout(KUBE_API_TIMEOUT, target_api.get_opt(secret_name)) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timeout checking image-pull Secret {secret_name} in {namespace}" + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + if existing.as_ref().is_some_and(|existing| { + !image_pull_secret_owned_by_gateway( + existing.metadata.labels.as_ref(), + &self.config.gateway_id, + workspace, + ) + }) { + return Err(KubernetesDriverError::Precondition(format!( + "image-pull Secret {secret_name} in {namespace} is not owned by this gateway" + ))); + } + + let copy = image_pull_secret_copy( + secret_name, + namespace, + workspace, + &self.config.gateway_id, + source, + ); match tokio::time::timeout( KUBE_API_TIMEOUT, target_api.patch( @@ -1554,6 +1623,14 @@ impl KubernetesComputeDriver { } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), tonic::Status> { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + )?; + self.config + .resource_admission + .validate() + .map_err(tonic::Status::failed_precondition)?; let _ = Self::validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; match self.config.workspace_mode { @@ -1582,6 +1659,88 @@ impl KubernetesComputeDriver { Ok(()) } + async fn admit_requested_resources( + &self, + sandbox: &Sandbox, + ) -> Result { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + )?; + self.validate_workspace_namespace(&sandbox.workspace) + .map_err(|error| tonic::Status::failed_precondition(error.to_string()))?; + let namespace = self + .config + .namespace_for_workspace(&sandbox.workspace, self.operator_allowlist.as_ref()) + .map_err(tonic::Status::failed_precondition)?; + let params = SandboxPodParams { + default_image: &self.config.default_image, + image_pull_secrets: &self.config.image_pull_secrets, + default_runtime_class_name: &self.config.default_runtime_class_name, + service_account_name: &self.config.service_account_name, + ..Default::default() + }; + let rendered = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) + .map_err(tonic::Status::invalid_argument)?; + crate::resource_admission::admit( + &self.client, + &self.config.resource_admission, + &sandbox.workspace, + &namespace, + &rendered["spec"]["podTemplate"]["spec"], + params.sandbox_secret_name, + ) + .await + } + + async fn admit_stored_resources(&self, object: &DynamicObject) -> Result<(), tonic::Status> { + if self.config.allow_driver_config && !self.config.resource_admission.enabled { + return Ok(()); + } + let expected = crate::resource_admission::check_record( + object.metadata.annotations.as_ref(), + self.config.allow_driver_config, + )?; + if !self.config.resource_admission.enabled { + return Ok(()); + } + let workspace = object + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_WORKSPACE)) + .ok_or_else(|| { + tonic::Status::failed_precondition("sandbox lacks workspace identity") + })?; + let namespace = object + .metadata + .namespace + .as_deref() + .ok_or_else(|| tonic::Status::failed_precondition("sandbox lacks namespace"))?; + sandbox_id_from_object(object).map_err(tonic::Status::failed_precondition)?; + let spec = &object.data["spec"]["podTemplate"]["spec"]; + let private_secret = sandbox_bootstrap_secret_name(spec).ok_or_else(|| { + tonic::Status::failed_precondition( + "sandbox pod template is missing its bootstrap Secret volume", + ) + })?; + let actual = crate::resource_admission::admit( + &self.client, + &self.config.resource_admission, + workspace, + namespace, + spec, + private_secret, + ) + .await?; + if actual != expected { + return Err(tonic::Status::failed_precondition( + "sandbox external resource identity or attachment inventory changed", + )); + } + Ok(()) + } + pub async fn get_sandbox(&self, sandbox_id: &str) -> Result, String> { info!( sandbox_id = %sandbox_id, @@ -1719,6 +1878,11 @@ impl KubernetesComputeDriver { &self, sandbox: &Sandbox, ) -> Result { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + ) + .map_err(admission_error)?; let gpu_requirements = sandbox .spec .as_ref() @@ -1739,7 +1903,8 @@ impl KubernetesComputeDriver { WorkspaceMode::Shared => self.config.namespace.clone(), WorkspaceMode::Managed => { let namespace = self.ensure_namespace(workspace).await?; - self.ensure_image_pull_secrets(&namespace).await?; + self.ensure_image_pull_secrets(&namespace, workspace) + .await?; namespace } WorkspaceMode::Operator => { @@ -1757,6 +1922,10 @@ impl KubernetesComputeDriver { if self.config.is_multi_namespace() { self.ensure_tls_secret(&target_namespace).await?; } + let resource_identities = self + .admit_requested_resources(sandbox) + .await + .map_err(admission_error)?; info!( sandbox_id = %sandbox.id, @@ -1818,6 +1987,21 @@ impl KubernetesComputeDriver { } let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); let mut annotations = sandbox_annotations(sandbox); + annotations.insert( + crate::resource_admission::IDENTITIES.into(), + serde_json::to_string(&resource_identities) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, + ); + annotations.insert( + crate::resource_admission::CONFIG_USED.into(), + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .and_then(|template| template.driver_config.as_ref()) + .is_some_and(|config| !config.fields.is_empty()) + .to_string(), + ); add_trace_context_annotation(&mut annotations); annotations.insert( ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING.to_string(), @@ -2336,6 +2520,14 @@ impl KubernetesComputeDriver { let workload_pod = self .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) .await?; + let admission_object = sandbox_api + .api + .get(cr_name) + .await + .map_err(KubernetesDriverError::from_kube)?; + self.admit_stored_resources(&admission_object) + .await + .map_err(admission_error)?; Self::validate_capability_free_workload_pod( &workload_pod, cr_uid, @@ -2607,6 +2799,14 @@ impl KubernetesComputeDriver { let workload_pod = self .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) .await?; + let admission_object = sandbox_api + .api + .get(cr_name) + .await + .map_err(KubernetesDriverError::from_kube)?; + self.admit_stored_resources(&admission_object) + .await + .map_err(admission_error)?; Self::validate_capability_free_workload_pod( &workload_pod, cr_uid, @@ -2929,6 +3129,9 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::from_kube)? .items; let mut object = select_expected_sandbox_runtime(objects, &expected_runtime_identity)?; + self.admit_stored_resources(&object) + .await + .map_err(admission_error)?; if sandbox_runtime_bootstrap_in_progress(&object) { let phase = sandbox_runtime_bootstrap_phase(&object); if phase != Some(SandboxRuntimeBootstrapPhase::Suspending) @@ -2981,6 +3184,9 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::from_kube)? .items; object = select_expected_sandbox_runtime(refreshed, &expected_runtime_identity)?; + self.admit_stored_resources(&object) + .await + .map_err(admission_error)?; } let namespace = object .metadata @@ -3562,6 +3768,14 @@ impl KubernetesComputeDriver { let Ok(sandbox_id) = sandbox_id_from_object(&object) else { continue; }; + if let Err(error) = self.admit_stored_resources(&object).await { + warn!(%sandbox_id, reason = %error.message(), "Sandbox resource admission revalidation failed"); + if error.code() == tonic::Code::FailedPrecondition { + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + } + continue; + } let namespace = object .metadata .namespace @@ -4668,15 +4882,25 @@ fn managed_ssh_network_policy(namespace: &str, config: &KubernetesComputeConfig) } } -fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> Secret { +fn image_pull_secret_copy( + secret_name: &str, + namespace: &str, + workspace: &str, + gateway_id: &str, + source: Secret, +) -> Secret { + let mut labels = source.metadata.labels.unwrap_or_default(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_GATEWAY_ID.to_string(), gateway_id.to_string()); + labels.insert(LABEL_SANDBOX_WORKSPACE.to_string(), workspace.to_string()); Secret { metadata: ObjectMeta { name: Some(secret_name.to_string()), namespace: Some(namespace.to_string()), - labels: Some(BTreeMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), + labels: Some(labels), ..Default::default() }, data: source.data, @@ -4685,6 +4909,22 @@ fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> } } +fn image_pull_secret_owned_by_gateway( + labels: Option<&BTreeMap>, + gateway_id: &str, + workspace: &str, +) -> bool { + labels.is_some_and(|labels| { + labels.get(LABEL_MANAGED_BY).map(String::as_str) == Some(LABEL_MANAGED_BY_VALUE) + && labels + .get(LABEL_GATEWAY_ID) + .is_none_or(|value| value == gateway_id) + && labels + .get(LABEL_SANDBOX_WORKSPACE) + .is_none_or(|value| value == workspace) + }) +} + fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { let mut annotations = BTreeMap::new(); annotations.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); @@ -5479,6 +5719,15 @@ const SANDBOX_PROXY_CA_VOLUME_NAME: &str = "openshell-run"; const SANDBOX_PROXY_CA_MOUNT_PATH: &str = "/run"; const SANDBOX_BOOTSTRAP_SCHEDULING_GATE: &str = "openshell.ai/bootstrap"; +fn sandbox_bootstrap_secret_name(spec: &serde_json::Value) -> Option<&str> { + spec["volumes"] + .as_array()? + .iter() + .find(|volume| volume["name"] == SANDBOX_BOOTSTRAP_VOLUME_NAME)?["secret"]["secretName"] + .as_str() + .filter(|name| !name.is_empty()) +} + /// Render the workload Pod that runs the `OpenShell` sandbox runtime. /// /// The pod receives no gateway credential or endpoint. Its non-root sandbox @@ -7121,6 +7370,207 @@ fn spawn_namespace_file_watcher( #[cfg(test)] mod tests { use super::*; + + #[tokio::test] + async fn admission_defaults_reject_driver_config_before_kubernetes_io_in_every_mode() { + for workspace_mode in [ + WorkspaceMode::Shared, + WorkspaceMode::Managed, + WorkspaceMode::Operator, + ] { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { + workspace_mode, + ..Default::default() + }); + let sandbox = Sandbox { + workspace: "team-a".into(), + name: "attacker".into(), + spec: Some(SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct( + serde_json::json!({"volumes":[{"name":"data","persistent_volume_claim":{"claim_name":"openshell-data-openshell-0"}}]}), + )), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let error = driver.validate_sandbox_create(&sandbox).await.unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("allow_driver_config")); + assert!( + matches!(driver.create_sandbox_inner(&sandbox).await, Err(KubernetesDriverError::Precondition(message)) if message.contains("allow_driver_config")) + ); + } + } + + #[tokio::test] + async fn admission_allows_private_default_workload_without_external_lookups() { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + let sandbox = Sandbox { + workspace: "team-a".into(), + name: "private".into(), + spec: Some(SandboxSpec { + template: Some(SandboxTemplate { + image: "test".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + assert!( + driver + .admit_requested_resources(&sandbox) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn admission_revalidates_stopped_sandbox_without_runtime_generation() { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut sandbox = DynamicObject::new("stopped-sandbox", &resource); + sandbox.metadata.namespace = Some("openshell".to_string()); + sandbox.metadata.labels = Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "sandbox-id".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ])); + sandbox.metadata.annotations = Some(BTreeMap::from([ + ( + crate::resource_admission::CONFIG_USED.to_string(), + "false".to_string(), + ), + ( + crate::resource_admission::IDENTITIES.to_string(), + "{}".to_string(), + ), + ])); + sandbox.data = serde_json::json!({ + "spec": { + "podTemplate": { + "spec": { + "automountServiceAccountToken": false, + "volumes": [{ + "name": SANDBOX_BOOTSTRAP_VOLUME_NAME, + "secret": {"secretName": "os-sandbox-sandbox-id-oldgeneration"} + }] + } + } + } + }); + + driver + .admit_stored_resources(&sandbox) + .await + .expect("stopped sandbox should use its stored private Secret reference"); + } + + #[tokio::test] + async fn admission_reconcile_does_not_suspend_on_forbidden_metadata_lookup() { + let sandbox = serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", + "kind": "Sandbox", + "metadata": { + "name": "sandbox-cr", + "namespace": "openshell", + "resourceVersion": "42", + "labels": { + LABEL_SANDBOX_ID: "sandbox-id", + LABEL_SANDBOX_WORKSPACE: "team-a" + }, + "annotations": { + crate::resource_admission::CONFIG_USED: "false", + crate::resource_admission::IDENTITIES: + "{\"PersistentVolumeClaim/openshell/team-data\":\"pvc-uid\"}" + } + }, + "spec": { + "podTemplate": { + "spec": { + "automountServiceAccountToken": false, + "volumes": [ + { + "name": "data", + "persistentVolumeClaim": {"claimName": "team-data"} + }, + { + "name": SANDBOX_BOOTSTRAP_VOLUME_NAME, + "secret": {"secretName": "os-sandbox-sandbox-id-generation"} + } + ] + } + } + } + }); + let steps = Arc::new(std::sync::Mutex::new(VecDeque::from([ + ( + http::Method::GET, + "/apis/agents.x-k8s.io/v1beta1/namespaces/openshell/sandboxes", + kube_test_response( + http::StatusCode::OK, + serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", + "kind": "SandboxList", + "items": [sandbox] + }), + ), + ), + ( + http::Method::GET, + "/api/v1/namespaces/openshell/persistentvolumeclaims/team-data", + kube_test_response( + http::StatusCode::FORBIDDEN, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Status", + "status": "Failure", + "message": "fixture forbidden", + "reason": "Forbidden", + "code": 403 + }), + ), + ), + ]))); + let service_steps = steps.clone(); + let service = tower::service_fn(move |request: http::Request| { + let steps = service_steps.clone(); + async move { + let (method, path, response) = steps + .lock() + .unwrap() + .pop_front() + .expect("403 admission retry must not issue a suspension patch"); + assert_eq!(request.method(), method); + assert_eq!(request.uri().path(), path); + Ok::<_, std::convert::Infallible>(response) + } + }); + let client = Client::new(service, "openshell"); + let driver = KubernetesComputeDriver { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + config: KubernetesComputeConfig::default(), + operator_allowlist: None, + }; + driver + .sandbox_api_version + .set(SANDBOX_VERSION_V1BETA1) + .expect("set test Sandbox API version"); + + driver.reconcile_sandbox_runtime_resources().await; + + assert!(steps.lock().unwrap().is_empty()); + } + use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, PROGRESS_COMPLETE_STEP_KEY, @@ -10504,7 +10954,7 @@ mod tests { } #[test] - fn image_pull_secret_copy_keeps_only_portable_secret_fields() { + fn image_pull_secret_copy_keeps_portable_fields_and_records_ownership() { let source: Secret = serde_json::from_value(serde_json::json!({ "apiVersion": "v1", "kind": "Secret", @@ -10522,7 +10972,7 @@ mod tests { })) .unwrap(); - let copy = image_pull_secret_copy("regcred", "workspace", source); + let copy = image_pull_secret_copy("regcred", "workspace", "team-a", "gateway-a", source); assert_eq!(copy.metadata.name.as_deref(), Some("regcred")); assert_eq!(copy.metadata.namespace.as_deref(), Some("workspace")); assert_eq!( @@ -10544,12 +10994,212 @@ mod tests { .map(String::as_str), Some(LABEL_MANAGED_BY_VALUE) ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get(LABEL_GATEWAY_ID) + .map(String::as_str), + Some("gateway-a") + ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get(LABEL_SANDBOX_WORKSPACE) + .map(String::as_str), + Some("team-a") + ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get("source-only") + .map(String::as_str), + Some("true") + ); assert!(copy.metadata.uid.is_none()); assert!(copy.metadata.resource_version.is_none()); assert!(copy.metadata.annotations.is_none()); assert!(copy.metadata.finalizers.is_none()); } + #[test] + fn image_pull_secret_collision_requires_gateway_ownership() { + let legacy = BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )]); + assert!(image_pull_secret_owned_by_gateway( + Some(&legacy), + "gateway-a", + "team-a" + )); + + let owned = BTreeMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_GATEWAY_ID.to_string(), "gateway-a".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ]); + assert!(image_pull_secret_owned_by_gateway( + Some(&owned), + "gateway-a", + "team-a" + )); + assert!(!image_pull_secret_owned_by_gateway( + Some(&owned), + "gateway-b", + "team-a" + )); + assert!(!image_pull_secret_owned_by_gateway( + Some(&owned), + "gateway-a", + "team-b" + )); + assert!(!image_pull_secret_owned_by_gateway( + Some(&BTreeMap::new()), + "gateway-a", + "team-a" + )); + } + + #[tokio::test] + async fn managed_image_pull_secret_admits_source_before_copying() { + let source_path = "/api/v1/namespaces/openshell/secrets/regcred"; + let target_path = "/api/v1/namespaces/managed-team-a/secrets/regcred"; + let copied = serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "regcred", + "namespace": "managed-team-a", + "labels": { + "openshell.ai/sandbox-attachable": "true", + LABEL_MANAGED_BY: LABEL_MANAGED_BY_VALUE, + LABEL_GATEWAY_ID: "gateway-a", + LABEL_SANDBOX_WORKSPACE: "team-a" + } + }, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + }); + let steps = Arc::new(std::sync::Mutex::new(VecDeque::from([ + ( + http::Method::GET, + source_path, + kube_test_response( + http::StatusCode::OK, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "regcred", + "namespace": "openshell", + "labels": {"openshell.ai/sandbox-attachable": "true"} + }, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + }), + ), + ), + ( + http::Method::GET, + target_path, + kube_test_not_found("secrets", "regcred"), + ), + ( + http::Method::PATCH, + target_path, + kube_test_response(http::StatusCode::OK, copied), + ), + ]))); + let service_steps = steps.clone(); + let service = tower::service_fn(move |request: http::Request| { + let steps = service_steps.clone(); + async move { + let (method, path, response) = steps + .lock() + .unwrap() + .pop_front() + .expect("unexpected Kubernetes API request"); + assert_eq!(request.method(), method); + assert_eq!(request.uri().path(), path); + Ok::<_, std::convert::Infallible>(response) + } + }); + let client = Client::new(service, "openshell"); + let driver = KubernetesComputeDriver { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + config: KubernetesComputeConfig { + namespace: "openshell".into(), + gateway_id: "gateway-a".into(), + image_pull_secrets: vec!["regcred".into()], + ..Default::default() + }, + operator_allowlist: None, + }; + + driver + .ensure_image_pull_secrets("managed-team-a", "team-a") + .await + .expect("approved source should be copied"); + assert!(steps.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn managed_image_pull_secret_rejects_unapproved_source_without_copying() { + let service = tower::service_fn( + move |request: http::Request| async move { + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().path(), + "/api/v1/namespaces/openshell/secrets/regcred" + ); + Ok::<_, std::convert::Infallible>(kube_test_response( + http::StatusCode::OK, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "regcred", "namespace": "openshell"}, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + }), + )) + }, + ); + let client = Client::new(service, "openshell"); + let driver = KubernetesComputeDriver { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + config: KubernetesComputeConfig { + namespace: "openshell".into(), + gateway_id: "gateway-a".into(), + image_pull_secrets: vec!["regcred".into()], + ..Default::default() + }, + operator_allowlist: None, + }; + + let error = driver + .ensure_image_pull_secrets("managed-team-a", "team-a") + .await + .expect_err("unapproved source must not be copied"); + assert!(matches!(error, KubernetesDriverError::Precondition(_))); + assert!( + error.to_string().contains("Secret 'openshell/regcred'"), + "unexpected error: {error}" + ); + } + #[test] fn namespace_owned_with_correct_labels() { let labels = BTreeMap::from([ diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index bb861308a2..607f95d382 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -6,6 +6,7 @@ pub mod driver; pub mod grpc; pub mod isolation; pub mod otel_tracing; +mod resource_admission; mod sandbox_runtime; pub use config::{ diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 8522e58b40..fcb8faa1e6 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -21,6 +21,13 @@ use openshell_driver_kubernetes::{ #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { + /// Operator-owned JSON policy; omitted means driver config disabled and labels required. + #[arg( + long, + env = "OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON", + default_value = "{}" + )] + admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig, /// Public compute-driver Unix socket used by an external gateway. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] bind_socket: Option, @@ -238,6 +245,8 @@ async fn main() -> Result<()> { let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { + allow_driver_config: args.admission_config_json.allow_driver_config, + resource_admission: args.admission_config_json.resource_admission.clone(), workspace_mode: args.workspace_mode, gateway_id: args.gateway_id, namespace: args.sandbox_namespace, diff --git a/crates/openshell-driver-kubernetes/src/resource_admission.rs b/crates/openshell-driver-kubernetes/src/resource_admission.rs new file mode 100644 index 0000000000..fa288c4f80 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/resource_admission.rs @@ -0,0 +1,515 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Closed workload-reference inventory and metadata-only resource resolution. + +use kube::{ + Api, Client, + api::ApiResource, + core::{DynamicObject, GroupVersionKind}, +}; +use openshell_core::resource_admission::ResourceAdmissionConfig; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; +use tonic::Status; + +pub const IDENTITIES: &str = "openshell.ai/resource-admission-identities"; +pub const CONFIG_USED: &str = "openshell.ai/caller-driver-config-used"; +pub type Identities = BTreeMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Scope { + /// Data-bearing resources selected for one `OpenShell` workspace. + Workspace, + /// Operator infrastructure intentionally reusable across workspaces. + Shared, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct Reference { + kind: &'static str, + name: String, + scope: Scope, +} + +fn resource_description(reference: &Reference, namespace: &str, cluster: bool) -> String { + if cluster { + format!("{} '{}'", reference.kind, reference.name) + } else { + format!("{} '{}/{}'", reference.kind, namespace, reference.name) + } +} + +fn contextualize(status: Status, resource: &str) -> Status { + Status::new(status.code(), format!("{resource}: {}", status.message())) +} + +fn metadata_lookup_error(error: kube::Error, resource: &str) -> Status { + match error { + kube::Error::Api(response) if response.code == 404 => { + Status::failed_precondition(format!("{resource} does not exist")) + } + kube::Error::Api(response) if response.code == 403 => Status::unavailable(format!( + "gateway is forbidden from reading metadata for {resource} (Kubernetes 403); check gateway RBAC" + )), + kube::Error::Api(response) if response.code == 401 => Status::unavailable(format!( + "gateway authentication was rejected while reading metadata for {resource} (Kubernetes 401)" + )), + kube::Error::Api(response) => Status::unavailable(format!( + "Kubernetes API returned {} ({}) while reading metadata for {resource}", + response.code, response.reason + )), + kube::Error::RustlsTls(_) | kube::Error::TlsRequired => Status::unavailable(format!( + "Kubernetes API TLS failed while reading metadata for {resource}" + )), + kube::Error::Auth(_) => Status::unavailable(format!( + "Kubernetes client authentication failed while reading metadata for {resource}" + )), + kube::Error::HyperError(_) | kube::Error::Service(_) => Status::unavailable(format!( + "Kubernetes API connection failed while reading metadata for {resource}" + )), + _ => Status::unavailable(format!( + "Kubernetes client failed while reading metadata for {resource}" + )), + } +} + +fn reference(refs: &mut BTreeSet, kind: &'static str, name: Option<&str>, scope: Scope) { + if let Some(name) = name.filter(|name| !name.is_empty()) { + refs.insert(Reference { + kind, + name: name.into(), + scope, + }); + } +} + +fn inventory(spec: &Value, private_secret: &str) -> Result, Status> { + let deny = || { + Status::failed_precondition("workload contains an unsupported external resource attachment") + }; + let mut refs = BTreeSet::new(); + reference( + &mut refs, + "RuntimeClass", + spec["runtimeClassName"].as_str(), + Scope::Shared, + ); + reference( + &mut refs, + "PriorityClass", + spec["priorityClassName"].as_str(), + Scope::Shared, + ); + // With automatic and projected tokens prohibited, the workload gets no + // ServiceAccount credentials. Selecting the Pod's identity is not a grant. + if spec["automountServiceAccountToken"] != false + || spec + .get("resourceClaims") + .is_some_and(|v| v.as_array().is_none_or(|a| !a.is_empty())) + { + return Err(deny()); + } + for secret in spec["imagePullSecrets"].as_array().into_iter().flatten() { + reference(&mut refs, "Secret", secret["name"].as_str(), Scope::Shared); + } + for volume in spec["volumes"].as_array().into_iter().flatten() { + let object = volume.as_object().ok_or_else(deny)?; + let sources: Vec<_> = object.keys().filter(|key| key.as_str() != "name").collect(); + if sources.len() != 1 { + return Err(deny()); + } + match sources[0].as_str() { + "emptyDir" | "downwardAPI" => {} + "persistentVolumeClaim" => reference( + &mut refs, + "PersistentVolumeClaim", + volume["persistentVolumeClaim"]["claimName"].as_str(), + Scope::Workspace, + ), + "secret" => { + let name = volume["secret"]["secretName"].as_str(); + if name != Some(private_secret) { + reference(&mut refs, "Secret", name, Scope::Workspace); + } + } + "configMap" => reference( + &mut refs, + "ConfigMap", + volume["configMap"]["name"].as_str(), + Scope::Workspace, + ), + _ => return Err(deny()), + } + } + for field in ["containers", "initContainers", "ephemeralContainers"] { + for container in spec[field].as_array().into_iter().flatten() { + for env in container["envFrom"].as_array().into_iter().flatten() { + reference( + &mut refs, + "Secret", + env["secretRef"]["name"].as_str(), + Scope::Workspace, + ); + reference( + &mut refs, + "ConfigMap", + env["configMapRef"]["name"].as_str(), + Scope::Workspace, + ); + } + for env in container["env"].as_array().into_iter().flatten() { + reference( + &mut refs, + "Secret", + env["valueFrom"]["secretKeyRef"]["name"].as_str(), + Scope::Workspace, + ); + reference( + &mut refs, + "ConfigMap", + env["valueFrom"]["configMapKeyRef"]["name"].as_str(), + Scope::Workspace, + ); + } + for field in ["requests", "limits"] { + for (resource, _) in container["resources"][field] + .as_object() + .into_iter() + .flatten() + { + if resource.contains('/') && resource != "nvidia.com/gpu" { + return Err(deny()); + } + } + } + } + } + Ok(refs) +} + +/// Resolve references selected through the OpenShell-owned Pod template. +/// Kubernetes control-plane mutations of the eventual live Pod are outside the +/// workspace-user authorization boundary and are not inventoried here. +pub async fn admit( + client: &Client, + policy: &ResourceAdmissionConfig, + workspace: &str, + namespace: &str, + spec: &Value, + private_secret: &str, +) -> Result { + policy.validate().map_err(Status::failed_precondition)?; + if !policy.enabled { + return Ok(BTreeMap::new()); + } + let mut identities = BTreeMap::new(); + for reference in inventory(spec, private_secret)? { + let (group, version, plural, cluster) = match reference.kind { + "PersistentVolumeClaim" => ("", "v1", "persistentvolumeclaims", false), + "Secret" => ("", "v1", "secrets", false), + "ConfigMap" => ("", "v1", "configmaps", false), + "RuntimeClass" => ("node.k8s.io", "v1", "runtimeclasses", true), + "PriorityClass" => ("scheduling.k8s.io", "v1", "priorityclasses", true), + _ => unreachable!("closed resource inventory"), + }; + let resource = ApiResource::from_gvk_with_plural( + &GroupVersionKind::gvk(group, version, reference.kind), + plural, + ); + let api: Api = if cluster { + Api::all_with(client.clone(), &resource) + } else { + Api::namespaced_with(client.clone(), namespace, &resource) + }; + let description = resource_description(&reference, namespace, cluster); + let object = tokio::time::timeout( + std::time::Duration::from_secs(30), + api.get_metadata(&reference.name), + ) + .await + .map_err(|_| { + Status::unavailable(format!( + "Kubernetes API timed out while reading metadata for {description}" + )) + })? + .map_err(|error| metadata_lookup_error(error, &description))?; + let metadata = object.metadata; + if metadata.deletion_timestamp.is_some() { + return Err(Status::failed_precondition(format!( + "{description} is being deleted" + ))); + } + let uid = metadata.uid.filter(|uid| !uid.is_empty()).ok_or_else(|| { + Status::failed_precondition(format!("{description} has no Kubernetes UID")) + })?; + let labels = metadata + .labels + .as_ref() + .into_iter() + .flat_map(|labels| labels.iter()); + match reference.scope { + Scope::Workspace => policy + .admit(workspace, labels) + .map_err(|status| contextualize(status, &description))?, + Scope::Shared => policy + .admit_shared(labels) + .map_err(|status| contextualize(status, &description))?, + } + identities.insert( + format!( + "{}/{}/{}", + reference.kind, + if cluster { "" } else { namespace }, + reference.name + ), + uid, + ); + } + Ok(identities) +} + +pub fn check_record( + annotations: Option<&BTreeMap>, + allow_config: bool, +) -> Result { + let annotations = annotations.ok_or_else(|| { + Status::failed_precondition("sandbox lacks admission provenance; recreate it") + })?; + match annotations.get(CONFIG_USED).map(String::as_str) { + Some("false") => {} + Some("true") if allow_config => {} + _ => { + return Err(Status::failed_precondition( + "sandbox driver config is disabled or lacks admission provenance; recreate it", + )); + } + } + annotations + .get(IDENTITIES) + .ok_or_else(|| { + Status::failed_precondition("sandbox lacks resource identities; recreate it") + }) + .and_then(|value| { + serde_json::from_str(value) + .map_err(|_| Status::failed_precondition("invalid resource admission record")) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn pvc_admission_uses_resource_metadata_not_namespace_or_read_only() { + for (labels, allowed) in [ + (serde_json::json!({}), false), + ( + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/sandbox-attachable-workspace":"other"}), + false, + ), + ( + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/sandbox-attachable-workspace":"team-a"}), + true, + ), + ] { + for read_only in [false, true] { + let labels = labels.clone(); + let service = tower::service_fn( + move |request: http::Request| { + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().path(), + "/api/v1/namespaces/shared/persistentvolumeclaims/openshell-data-openshell-0" + ); + assert!( + request.headers()["accept"] + .to_str() + .unwrap() + .contains("PartialObjectMetadata") + ); + let body = serde_json::json!({"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata", + "metadata":{"name":"openshell-data-openshell-0","namespace":"shared","uid":"fixture-pvc","labels":labels}}); + async move { + Ok::<_, std::convert::Infallible>( + http::Response::builder() + .header("content-type", "application/json") + .body(kube::client::Body::from(body.to_string().into_bytes())) + .unwrap(), + ) + } + }, + ); + let client = Client::new(service, "shared"); + let spec = serde_json::json!({"automountServiceAccountToken":false,"volumes":[{ + "name":"data","persistentVolumeClaim":{"claimName":"openshell-data-openshell-0","readOnly":read_only}}]}); + let result = admit( + &client, + &ResourceAdmissionConfig::default(), + "team-a", + "shared", + &spec, + "private", + ) + .await; + assert_eq!(result.is_ok(), allowed, "{result:?}"); + if let Ok(identities) = result { + assert_eq!(identities.len(), 1); + } + } + } + } + + #[tokio::test] + async fn metadata_lookup_errors_identify_the_resource_and_failure_class() { + for (http_status, reason, expected_code, expected_message) in [ + ( + 404, + "NotFound", + tonic::Code::FailedPrecondition, + "does not exist", + ), + ( + 403, + "Forbidden", + tonic::Code::Unavailable, + "check gateway RBAC", + ), + ( + 503, + "ServiceUnavailable", + tonic::Code::Unavailable, + "Kubernetes API returned 503 (ServiceUnavailable)", + ), + ] { + let service = tower::service_fn( + move |_request: http::Request| async move { + let body = serde_json::json!({ + "apiVersion": "v1", + "kind": "Status", + "status": "Failure", + "message": "fixture failure", + "reason": reason, + "code": http_status, + }); + Ok::<_, std::convert::Infallible>( + http::Response::builder() + .status(http_status) + .header("content-type", "application/json") + .body(kube::client::Body::from(body.to_string().into_bytes())) + .unwrap(), + ) + }, + ); + let client = Client::new(service, "shared"); + let spec = serde_json::json!({ + "automountServiceAccountToken": false, + "volumes": [{ + "name": "data", + "persistentVolumeClaim": {"claimName": "team-data"} + }] + }); + + let error = admit( + &client, + &ResourceAdmissionConfig::default(), + "team-a", + "shared", + &spec, + "private", + ) + .await + .expect_err("lookup must fail"); + + assert_eq!(error.code(), expected_code); + assert!( + error + .message() + .contains("PersistentVolumeClaim 'shared/team-data'"), + "unexpected error: {error}" + ); + assert!( + error.message().contains(expected_message), + "unexpected error: {error}" + ); + } + } + + #[tokio::test] + async fn label_denial_identifies_the_resource() { + let service = tower::service_fn(|_request: http::Request| async move { + let body = serde_json::json!({ + "apiVersion": "meta.k8s.io/v1", + "kind": "PartialObjectMetadata", + "metadata": { + "name": "kata", + "uid": "runtime-class-uid", + "labels": {} + } + }); + Ok::<_, std::convert::Infallible>( + http::Response::builder() + .header("content-type", "application/json") + .body(kube::client::Body::from(body.to_string().into_bytes())) + .unwrap(), + ) + }); + let client = Client::new(service, "shared"); + let spec = serde_json::json!({ + "automountServiceAccountToken": false, + "runtimeClassName": "kata" + }); + + let error = admit( + &client, + &ResourceAdmissionConfig::default(), + "team-a", + "shared", + &spec, + "private", + ) + .await + .expect_err("unlabelled RuntimeClass must be denied"); + + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert_eq!( + error.message(), + "RuntimeClass 'kata': external resource not admitted by required labels" + ); + } + + #[test] + fn inventories_all_containers_and_reference_aliases() { + let pod = serde_json::json!({"automountServiceAccountToken":false,"runtimeClassName":"r","priorityClassName":"p", + "volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"gateway-db","readOnly":true}}], + "initContainers":[{"envFrom":[{"secretRef":{"name":"secret"}}]}], + "containers":[{"env":[{"valueFrom":{"configMapKeyRef":{"name":"config"}}}]}]}); + let refs = inventory(&pod, "private").unwrap(); + assert_eq!(refs.len(), 5); + assert!( + refs.iter() + .any(|r| r.name == "gateway-db" && r.scope == Scope::Workspace) + ); + assert!( + refs.iter() + .any(|r| r.name == "r" && r.scope == Scope::Shared) + ); + } + #[test] + fn rejects_unsupported_volume_sources_but_allows_gpu() { + for kind in ["hostPath", "csi", "projected", "image"] { + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"volumes":[{"name":"x",kind:{}}]}), "private").is_err()); + } + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"containers":[{"resources":{"limits":{"nvidia.com/gpu":"1"}}}]}), "private").is_ok()); + } + #[test] + fn legacy_and_forbidden_config_records_fail_closed() { + assert!(check_record(None, true).is_err()); + let record = BTreeMap::from([ + (CONFIG_USED.into(), "true".into()), + (IDENTITIES.into(), "{}".into()), + ]); + assert!(check_record(Some(&record), false).is_err()); + assert!(check_record(Some(&record), true).is_ok()); + } +} diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 9c235b85c7..b4ee82abb2 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -4,6 +4,12 @@ OpenShell compute driver backed by **Microsoft MXC** (`wxc-exec`) on Windows. ## Design +Caller driver config is disabled by default, so command-based MXC workflows +need explicit administrator opt-in. Host filesystem grants have no trusted +label resolver and are rejected while resource admission is enabled. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission) +for the independent controls and the security consequences of opting out. + This driver implements the gateway's ordinary in-process `ComputeDriver` contract and is linked into `openshell-gateway`. It sets `driver_reports_runtime_readiness`, so the gateway accepts driver-reported diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index 23491533bb..31692d659f 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -59,6 +59,10 @@ pub enum MxcBackend { // configuration schema rather than one compound state machine. #[allow(clippy::struct_excessive_bools)] pub struct MxcComputeConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// Path to `wxc-exec.exe`. Required for live runs. pub wxc_exec_path: String, /// Backend to target. Default: `process_container`. @@ -91,6 +95,9 @@ impl Default for MxcComputeConfig { fn default() -> Self { Self { wxc_exec_path: "wxc-exec.exe".into(), + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), backend: MxcBackend::default(), pc_least_privilege: false, pc_capabilities: Vec::new(), @@ -453,6 +460,11 @@ impl MxcComputeBackend { pub fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: DRIVER_NAME.to_string(), driver_version: DRIVER_VERSION.to_string(), default_image: DEFAULT_IMAGE_SENTINEL.to_string(), @@ -513,6 +525,19 @@ impl MxcComputeBackend { } pub fn validate_sandbox_create(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { + self.config + .resource_admission + .validate() + .map_err(tonic::Status::failed_precondition)?; + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + )?; + // MXC grants access to existing host filesystem objects, including its + // executable/workdir. There is no authoritative label resolver yet. + self.config + .resource_admission + .reject_unlabelable("MXC host filesystem grants")?; Self::validate_sandbox_fields(sandbox)?; let policy = sandbox.spec.as_ref().and_then(|spec| spec.policy.as_ref()); let egress_addr = configured_egress_addr(&self.config)?; @@ -533,6 +558,7 @@ impl MxcComputeBackend { } pub async fn create_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { + self.validate_sandbox_create(sandbox)?; let sandbox_id = sandbox.id.clone(); Self::validate_sandbox_fields(sandbox)?; @@ -1232,6 +1258,30 @@ mod lifecycle_tests { }; use std::time::Duration; + fn host_grants_config() -> MxcComputeConfig { + MxcComputeConfig { + allow_driver_config: true, + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig { + enabled: false, + ..Default::default() + }, + ..Default::default() + } + } + + #[tokio::test] + async fn admission_rejects_host_grants_even_with_driver_config_enabled() { + let backend = MxcComputeBackend::new_mocked(MxcComputeConfig { + allow_driver_config: true, + ..Default::default() + }); + let sandbox = driver_sandbox("sb-admission"); + let error = backend.validate_sandbox_create(&sandbox).unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(backend.create_sandbox(&sandbox).await.is_err()); + assert!(backend.get_sandbox("sb-admission").await.is_none()); + } + fn driver_sandbox(id: &str) -> DriverSandbox { driver_sandbox_with_command(id, "", vec!["cmd".into(), "/c".into(), "exit 0".into()]) } @@ -1422,7 +1472,7 @@ mod lifecycle_tests { "-Command".into(), format!("Set-Content -LiteralPath {hello} -Value hi"), ]; - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); let policy = fs_policy(&[&share]); let sb = with_policy(driver_sandbox_with_command("sb-pos", &share, cmd), policy); @@ -1483,7 +1533,7 @@ mod lifecycle_tests { "-Command".into(), format!("Set-Content -LiteralPath {hello} -Value hi"), ]; - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); let policy = fs_policy(&[&share]); let sb = with_policy(driver_sandbox_with_command("sb-pc", &share, cmd), policy); @@ -1535,7 +1585,7 @@ mod lifecycle_tests { backend: MxcBackend::ProcessContainer, egress_proxy: true, egress_proxy_addr: "127.0.0.1:18080".into(), - ..Default::default() + ..host_grants_config() }; let backend = MxcComputeBackend::new_mocked(config); let mut stream = backend.watch_sandboxes().await; @@ -1644,7 +1694,7 @@ mod lifecycle_tests { "-Command".into(), format!("Set-Content -LiteralPath {out_path} -Value hi"), ]; - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); // Subscribe to the watch stream BEFORE create so we catch the denial event. let mut stream = backend.watch_sandboxes().await; @@ -1703,7 +1753,7 @@ mod lifecycle_tests { "-Command".into(), format!("$null = '{share}'; Start-Sleep -Seconds 60"), ]; - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); let policy = fs_policy(&[&share]); let sandbox = with_policy(driver_sandbox_with_command("sb-stop", "", command), policy); backend @@ -1733,7 +1783,7 @@ mod lifecycle_tests { let tmp = tempfile::tempdir().unwrap(); let share = tmp.path().to_string_lossy().replace('\\', "/"); - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); let mut policy = fs_policy(&[&share]); policy.network_policies.insert( @@ -1761,7 +1811,7 @@ mod lifecycle_tests { let config = MxcComputeConfig { egress_proxy: true, egress_proxy_addr: "127.0.0.1:18080".into(), - ..Default::default() + ..host_grants_config() }; let backend = MxcComputeBackend::new_mocked(config); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 85cdb33259..5147d78a51 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -13,6 +13,13 @@ identity, DNS, TCP, and loopback-forwarding semantics. ## Runtime posture +Caller driver config is disabled by default. Existing volumes require +administrator-controlled approval labels; bind and supplemental image mounts +are denied under enforcement. Private-volume names alone do not prove +ownership. GPU devices are temporarily exempt. Admission runs before launch, +restart, and periodically for running workloads. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). + | Property | Workload | Supervisor | |---|---|---| | UID/GID | Pinned non-root workload identity | Same mapped identity | @@ -107,7 +114,9 @@ requires the authenticated supervisor session before publishing Ready. User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain native Podman features and apply only to the workload. Bind mounts require the -operator's `enable_bind_mounts` opt-in. Reserved control paths and the workspace +operator's `enable_bind_mounts` opt-in and disabled label admission. Supplemental +image mounts also require disabled admission. Driver JSON requires +`allow_driver_config = true`. Reserved control paths and the workspace root cannot be replaced. User-owned volumes are never created or deleted. See [gateway configuration](../../docs/reference/gateway-config.mdx) for diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 8bf332d296..80e24513a3 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -88,6 +88,8 @@ pub fn validate_name(name: &str) -> Result<(), PodmanApiError> { #[derive(Debug, Clone, serde::Deserialize)] #[serde(rename_all = "PascalCase")] pub struct ContainerInspect { + #[serde(default)] + pub mounts: Option>, pub id: String, pub name: String, pub state: ContainerState, @@ -223,12 +225,24 @@ pub struct PortMappingEntry { #[derive(Debug, Clone, Default, serde::Deserialize)] #[serde(rename_all = "PascalCase")] pub struct VolumeInspect { + #[serde(default)] + pub created_at: Option, + #[serde(default)] + pub labels: Option>, + #[serde(default)] + pub name: String, #[serde(default)] pub driver: String, #[serde(default)] pub options: HashMap, } +impl VolumeInspect { + pub(crate) fn admission_identity(&self) -> Value { + serde_json::json!({"name": self.name, "driver": self.driver, "options": self.options, "created_at": self.created_at}) + } +} + /// A Podman event from the events stream. #[derive(Debug, Clone, serde::Deserialize)] #[serde(rename_all = "PascalCase")] @@ -686,11 +700,53 @@ impl PodmanClient { // ── Volume operations ──────────────────────────────────────────────── - /// Create a named volume. Idempotent (conflict is ignored). - pub async fn create_volume(&self, name: &str) -> Result<(), PodmanApiError> { - validate_name(name)?; - self.create_ignore_conflict("/libpod/volumes/create", &serde_json::json!({"Name": name})) - .await + /// Never adopt an unrelated existing volume on a private provisioning path. + pub(crate) async fn create_owned_volume( + &self, + name: &str, + sandbox_id: &str, + workspace: &str, + ) -> Result<(), PodmanApiError> { + let labels = HashMap::from([ + ( + openshell_core::driver_utils::LABEL_SANDBOX_ID.to_string(), + sandbox_id.to_string(), + ), + ( + openshell_core::driver_utils::LABEL_SANDBOX_WORKSPACE.to_string(), + workspace.to_string(), + ), + ]); + match self.inspect_volume(name).await { + Ok(existing) => { + if existing.driver != "local" + || !existing.options.is_empty() + || existing.labels.as_ref() != Some(&labels) + { + return Err(PodmanApiError::InvalidInput( + "private volume name collides with an unrelated resource".into(), + )); + } + return Ok(()); + } + Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error), + } + self.create_ignore_conflict( + "/libpod/volumes/create", + &serde_json::json!({"Name":name,"Driver":"local","Labels":labels}), + ) + .await?; + let created = self.inspect_volume(name).await?; + if created.driver != "local" + || !created.options.is_empty() + || created.labels.as_ref() != Some(&labels) + { + return Err(PodmanApiError::InvalidInput( + "private volume ownership verification failed".into(), + )); + } + Ok(()) } /// Remove a named volume. Idempotent (not-found is ignored). diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 3812367234..75dc2aa77e 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -27,6 +27,10 @@ pub const fn podman_image_pull_policy(policy: ImagePullPolicy) -> &'static str { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] pub struct PodmanComputeConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// Podman API Unix socket. When unset, use the socket selected by /// gateway auto-detection. pub socket_path: Option, @@ -224,6 +228,9 @@ pub fn parse_id_map_entry( impl PodmanComputeConfig { /// Validate and normalize startup configuration without connecting to Podman. pub fn validate_configuration(&mut self) -> Result<(), crate::client::PodmanApiError> { + self.resource_admission + .validate() + .map_err(crate::client::PodmanApiError::InvalidInput)?; self.validate_tls_config()?; self.validate_runtime_limits()?; self.validate_host_gateway_ip()?; @@ -462,6 +469,9 @@ impl Default for PodmanComputeConfig { fn default() -> Self { Self { socket_path: None, + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), default_image: openshell_core::image::default_sandbox_image(), image_pull_policy: ImagePullPolicy::default(), grpc_endpoint: String::new(), diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index e25c30a1fb..1c7d0f62db 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -85,6 +85,23 @@ pub struct PodmanSandboxDriverConfig { } impl PodmanSandboxDriverConfig { + pub(crate) fn admit_mount_types( + &self, + policy: &openshell_core::resource_admission::ResourceAdmissionConfig, + ) -> Result<(), ComputeDriverError> { + for mount in &self.mounts { + if matches!( + mount, + PodmanDriverMountConfig::Bind { .. } | PodmanDriverMountConfig::Image { .. } + ) { + policy + .reject_unlabelable("host bind or image mount") + .map_err(|error| ComputeDriverError::Precondition(error.message().into()))?; + } + } + Ok(()) + } + pub fn from_sandbox(sandbox: &DriverSandbox) -> Result { let Some(template) = sandbox .spec @@ -653,6 +670,13 @@ fn build_labels(sandbox: &DriverSandbox) -> BTreeMap { } } // Managed labels (highest priority -- always overwrite). + labels.insert( + openshell_core::resource_admission::CONFIG_USED_LABEL.into(), + template + .and_then(|t| t.driver_config.as_ref()) + .is_some_and(|config| !config.fields.is_empty()) + .to_string(), + ); labels.insert(LABEL_SANDBOX_ID.into(), sandbox.id.clone()); labels.insert(LABEL_SANDBOX_NAME.into(), sandbox.name.clone()); labels.insert(LABEL_SANDBOX_NAMESPACE.into(), sandbox.namespace.clone()); @@ -1083,7 +1107,13 @@ fn build_base_spec( let vol = volume_name(&sandbox.id); let env = build_env(sandbox, config, requested_image, oci_user)?; - let labels = build_labels(sandbox); + let mut labels = build_labels(sandbox); + labels.insert( + "openshell.ai/runtime-binary-source".into(), + supervisor_bin_path + .map(|path| path.display().to_string()) + .unwrap_or_default(), + ); let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) .map_err(ComputeDriverError::InvalidArgument)?; @@ -1411,6 +1441,20 @@ pub struct IsolationSpecs { pub supervisor: ContainerSpec, } +impl IsolationSpecs { + pub(crate) fn record_resource_identities( + &mut self, + identities: &BTreeMap, + ) -> Result<(), ComputeDriverError> { + self.workload.labels.insert( + openshell_core::resource_admission::IDENTITIES_LABEL.into(), + serde_json::to_string(identities) + .map_err(|error| ComputeDriverError::Message(error.to_string()))?, + ); + Ok(()) + } +} + pub fn build_isolation_specs( input: IsolationSpecInput<'_>, ) -> Result { diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 2c9303f1c1..335a9ef2e3 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -507,7 +507,7 @@ impl PodmanComputeDriver { ); } - Ok(Self { + let driver = Self { client, config, rootless, @@ -517,12 +517,25 @@ impl PodmanComputeDriver { )), gpu_inventory_refresh: Arc::new(local_podman_gpu_selector_state), lifecycle_event_fences: LifecycleEventFences::default(), - }) + }; + let reconciler = driver.clone(); + tokio::spawn(async move { + loop { + reconciler.reconcile_resource_admission().await; + tokio::time::sleep(Duration::from_secs(30)).await; + } + }); + Ok(driver) } /// Report driver capabilities. pub fn capabilities(&self) -> Result { Ok(GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: "podman".to_string(), driver_version: openshell_core::VERSION.to_string(), default_image: self.config.default_image.clone(), @@ -570,12 +583,18 @@ impl PodmanComputeDriver { &self, sandbox: &'a DriverSandbox, ) -> Result, ComputeDriverError> { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + ) + .map_err(|error| ComputeDriverError::Precondition(error.message().into()))?; let gpu_requirements = sandbox .spec .as_ref() .and_then(|spec| spec.resource_requirements.as_ref()) .and_then(|requirements| driver_gpu_requirements(Some(requirements))); let driver_config = PodmanSandboxDriverConfig::from_sandbox(sandbox)?; + driver_config.admit_mount_types(&self.config.resource_admission)?; Self::validate_gpu_request(gpu_requirements, &driver_config)?; self.validate_user_volume_mounts_available(sandbox).await?; let _ = self.resolve_gpu_cdi_devices( @@ -646,13 +665,31 @@ impl PodmanComputeDriver { async fn validate_user_volume_mounts_available( &self, sandbox: &DriverSandbox, - ) -> Result<(), ComputeDriverError> { + ) -> Result, ComputeDriverError> { + let mut identities = std::collections::BTreeMap::new(); let volumes = container::podman_driver_volume_mount_sources(sandbox, self.config.enable_bind_mounts) .map_err(ComputeDriverError::Precondition)?; for volume in volumes { match self.client.inspect_volume(&volume).await { Ok(volume_info) => { + identities.insert(volume.clone(), volume_info.admission_identity()); + self.config + .resource_admission + .admit( + &sandbox.workspace, + volume_info + .labels + .as_ref() + .into_iter() + .flat_map(|labels| labels.iter()), + ) + .map_err(|error| { + ComputeDriverError::Precondition(format!( + "podman volume '{volume}': {}", + error.message() + )) + })?; if !self.config.enable_bind_mounts && podman_volume_is_bind_backed(&volume_info) { return Err(ComputeDriverError::Precondition(format!( @@ -668,9 +705,144 @@ impl PodmanComputeDriver { Err(err) => return Err(ComputeDriverError::from(err)), } } + Ok(identities) + } + + /// Create a sandbox container. + async fn admit_container_resources(&self, id: &str) -> Result<(), ComputeDriverError> { + if self.config.allow_driver_config && !self.config.resource_admission.enabled { + return Ok(()); + } + let inspect = self.client.inspect_container(id).await?; + let labels = &inspect.config.labels; + let precondition = + |error: tonic::Status| ComputeDriverError::Precondition(error.message().into()); + openshell_core::resource_admission::check_config_provenance( + self.config.allow_driver_config, + labels + .get(openshell_core::resource_admission::CONFIG_USED_LABEL) + .map(String::as_str), + ) + .map_err(precondition)?; + if !self.config.resource_admission.enabled { + return Ok(()); + } + let missing = + || ComputeDriverError::Precondition("sandbox lacks attachment provenance".into()); + let workspace = labels + .get(container::LABEL_SANDBOX_WORKSPACE) + .ok_or_else(missing)?; + let sandbox_id = labels.get(LABEL_SANDBOX_ID).ok_or_else(missing)?; + let mounts = inspect.mounts.as_ref().ok_or_else(missing)?; + let expected: std::collections::BTreeMap = labels + .get(openshell_core::resource_admission::IDENTITIES_LABEL) + .and_then(|value| serde_json::from_str(value).ok()) + .ok_or_else(missing)?; + let mut actual = std::collections::BTreeMap::new(); + for mount in mounts { + match mount["Type"].as_str() { + Some("volume") => { + let name = mount["Name"].as_str().ok_or_else(missing)?; + let volume = + self.client + .inspect_volume(name) + .await + .map_err(|error| match error { + PodmanApiError::NotFound(_) => ComputeDriverError::Precondition( + "attached volume no longer exists".into(), + ), + other => ComputeDriverError::from(other), + })?; + if volume.name != name { + return Err(missing()); + } + if name == container::volume_name(sandbox_id) + || name == crate::isolation::channel_volume_name(sandbox_id) + { + let owned = volume.labels.as_ref().is_some_and(|labels| { + labels.get(LABEL_SANDBOX_ID) == Some(sandbox_id) + && labels.get(container::LABEL_SANDBOX_WORKSPACE) == Some(workspace) + }); + if !owned || volume.driver != "local" || !volume.options.is_empty() { + return Err(missing()); + } + } else { + actual.insert(name.to_string(), volume.admission_identity()); + self.config + .resource_admission + .admit( + workspace, + volume + .labels + .as_ref() + .into_iter() + .flat_map(|labels| labels.iter()), + ) + .map_err(|error| { + ComputeDriverError::Precondition(format!( + "podman volume '{name}': {}", + error.message() + )) + })?; + if !self.config.enable_bind_mounts && podman_volume_is_bind_backed(&volume) + { + return Err(ComputeDriverError::Precondition( + "bind-backed volume is disabled".into(), + )); + } + } + } + Some("tmpfs") => {} + Some("bind") + if mount["Destination"].as_str() + == Some(openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY) + && mount["RW"].as_bool() == Some(false) + && labels + .get("openshell.ai/runtime-binary-source") + .filter(|path| !path.is_empty()) + .map(String::as_str) + == mount["Source"].as_str() => {} + _ => self + .config + .resource_admission + .reject_unlabelable("effective Podman mount") + .map_err(precondition)?, + } + } + if actual != expected { + return Err(ComputeDriverError::Precondition( + "external volume identity or attachment inventory changed".into(), + )); + } Ok(()) } + /// Revalidate running grants every 30 seconds. Outages block launches but + /// only confirmed denials stop existing workloads. + async fn reconcile_resource_admission(&self) { + let Ok(entries) = self + .client + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) + .await + else { + return; + }; + for entry in entries.iter().filter(|entry| entry.state == "running") { + if let Err(ComputeDriverError::Precondition(reason)) = + self.admit_container_resources(&entry.id).await + { + warn!(container = %entry.id, %reason, "Stopping sandbox after resource admission denial"); + let _ = self.client.stop_container(&entry.id, 0).await; + if let Some(id) = entry.labels.get(LABEL_SANDBOX_ID) { + let _ = self + .client + .stop_container(&crate::isolation::supervisor_name(id), 0) + .await; + } + } + } + } + /// Create a sandbox container. #[tracing::instrument( name = "podman.provision", @@ -837,7 +1009,7 @@ impl PodmanComputeDriver { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { self.client - .create_volume(&vol_name) + .create_owned_volume(&vol_name, &sandbox.id, &sandbox.workspace) .await .map_err(ComputeDriverError::from)?; let token_secret_name = @@ -876,8 +1048,11 @@ impl PodmanComputeDriver { // Clean up the volume and both per-sandbox secrets on any failure past // this point. + let channel_owned = std::sync::atomic::AtomicBool::new(false); let cleanup_created = || async { - let _ = self.client.remove_volume(&channel_volume).await; + if channel_owned.load(std::sync::atomic::Ordering::Relaxed) { + let _ = self.client.remove_volume(&channel_volume).await; + } let _ = self.client.remove_volume(&vol_name).await; if let Some(secret) = token_secret_name.as_deref() { cleanup_sandbox_token_secret(&self.client, secret).await; @@ -946,7 +1121,7 @@ impl PodmanComputeDriver { identity: &identity, rootless: self.rootless, }); - let specs = match specs { + let mut specs = match specs { Ok(spec) => spec, Err(e) => { cleanup_all().await; @@ -956,10 +1131,16 @@ impl PodmanComputeDriver { let mut created_workload = None; let mut created_supervisor = None; let create_result = async { - self.client.create_volume(&channel_volume).await?; + let identities = self.validate_user_volume_mounts_available(sandbox).await?; + specs.record_resource_identities(&identities)?; + self.client + .create_owned_volume(&channel_volume, &sandbox.id, &sandbox.workspace) + .await?; + channel_owned.store(true, std::sync::atomic::Ordering::Relaxed); let workload_id = self.client.create_typed_container(&specs.workload).await?; created_workload = Some(workload_id.clone()); self.client.verify_isolation_fence(&workload_id).await?; + self.admit_container_resources(&workload_id).await?; let child_env = podman_child_environment(sandbox, &image_env); let launch_authentication = sandbox .spec @@ -1249,6 +1430,7 @@ impl PodmanComputeDriver { .find_container(sandbox_id) .await? .ok_or(ComputeDriverError::NotFound)?; + self.admit_container_resources(&container.id).await?; if container.state == "running" { let supervisor = self .client @@ -2181,7 +2363,7 @@ mod tests { let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; let (socket_path, requests, handle) = spawn_podman_stub( "trace-create", - create_setup_responses(false) + create_setup_responses(false, "sandbox-trace") .into_iter() .chain(create_launch_responses()) .collect(), @@ -2614,7 +2796,10 @@ mod tests { async fn validate_sandbox_create_passes_explicit_cdi_device_id_without_inventory() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; - let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig::default()); + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + allow_driver_config: true, + ..Default::default() + }); let sandbox = DriverSandbox { spec: Some(DriverSandboxSpec { resource_requirements: Some(gpu_resources(None)), @@ -2721,6 +2906,11 @@ mod tests { fn test_driver(socket_path: PathBuf) -> PodmanComputeDriver { let config = PodmanComputeConfig { + allow_driver_config: true, + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig { + enabled: false, + ..Default::default() + }, socket_path: Some(socket_path), stop_timeout_secs: 10, ..PodmanComputeConfig::default() @@ -2728,7 +2918,9 @@ mod tests { PodmanComputeDriver::for_tests(config) } - fn test_driver_with_config(config: PodmanComputeConfig) -> PodmanComputeDriver { + fn test_driver_with_config(mut config: PodmanComputeConfig) -> PodmanComputeDriver { + config.allow_driver_config = true; + config.resource_admission.enabled = false; PodmanComputeDriver::for_tests(config) } @@ -2763,6 +2955,97 @@ mod tests { } } + #[tokio::test] + async fn private_volume_collision_is_not_adopted_or_relabelled() { + let (socket, requests, handle) = spawn_podman_stub( + "admission-collision", + vec![StubResponse::new( + StatusCode::OK, + serde_json::json!({ + "Name":"private-collision", "Driver":"local", "Options":{}, "Labels":{} + }) + .to_string(), + )], + ); + let driver = test_driver(socket.clone()); + assert!( + driver + .client + .create_owned_volume("private-collision", "sandbox-123", "team-a") + .await + .is_err() + ); + handle.await.unwrap(); + let logged = requests.lock().unwrap(); + assert_eq!(logged.len(), 1); + assert!(logged[0].starts_with("GET ")); + let _ = fs::remove_file(socket); + } + + #[tokio::test] + async fn admission_requires_explicit_volume_labels_and_workspace_match() { + for (labels, allowed) in [ + (serde_json::json!(null), false), + (serde_json::json!({}), false), + ( + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/sandbox-attachable-workspace":"other"}), + false, + ), + ( + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/sandbox-attachable-workspace":"team-a"}), + true, + ), + ] { + let (socket, requests, handle) = spawn_podman_stub("admission-labels", vec![StubResponse::new(StatusCode::OK, + serde_json::json!({"Name":"existing","Driver":"local","Options":{},"Labels":labels}).to_string())]); + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + socket_path: Some(socket.clone()), + allow_driver_config: true, + ..Default::default() + }); + let mut sandbox = sandbox_with_volume_mount("existing"); + sandbox.workspace = "team-a".into(); + let result = driver.validate_sandbox_create(&sandbox).await; + if !allowed { + assert!( + result + .as_ref() + .unwrap_err() + .to_string() + .contains("podman volume 'existing'") + ); + } + assert_eq!(result.is_ok(), allowed); + handle.await.unwrap(); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|request| request.starts_with("GET ")) + ); + let _ = fs::remove_file(socket); + } + } + + #[tokio::test] + async fn admission_driver_config_denial_does_not_contact_podman() { + for enabled in [true, false] { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig { + enabled, + ..Default::default() + }, + ..Default::default() + }); + let error = driver + .validate_sandbox_create(&sandbox_with_volume_mount("existing")) + .await + .unwrap_err(); + assert!(error.to_string().contains("allow_driver_config")); + } + } + fn api_path(path: &str) -> String { format!("/v5.0.0{path}") } @@ -2770,6 +3053,9 @@ mod tests { #[test] fn podman_local_volume_with_bind_option_is_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: "local".to_string(), options: HashMap::from([("o".to_string(), "rw,bind".to_string())]), }; @@ -2780,6 +3066,9 @@ mod tests { #[test] fn podman_local_volume_with_rbind_option_is_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: "local".to_string(), options: HashMap::from([("o".to_string(), "rw,rbind".to_string())]), }; @@ -2790,6 +3079,9 @@ mod tests { #[test] fn podman_empty_driver_volume_with_bind_option_is_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: String::new(), options: HashMap::from([("o".to_string(), "bind".to_string())]), }; @@ -2800,6 +3092,9 @@ mod tests { #[test] fn podman_local_volume_without_bind_option_is_not_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: "local".to_string(), options: HashMap::from([("o".to_string(), "addr=127.0.0.1,rw".to_string())]), }; @@ -2810,6 +3105,9 @@ mod tests { #[test] fn podman_nonlocal_volume_with_bind_option_is_not_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: "custom".to_string(), options: HashMap::from([("o".to_string(), "bind".to_string())]), }; @@ -3164,7 +3462,7 @@ mod tests { StubResponse::new(StatusCode::OK, archive) } - fn create_setup_responses(proxy_secret: bool) -> Vec { + fn create_setup_responses(proxy_secret: bool, sandbox_id: &str) -> Vec { let mut responses = vec![ StubResponse::new(StatusCode::OK, "{}"), // sandbox runtime pull StubResponse::new(StatusCode::OK, "{}"), // supervisor pull @@ -3177,7 +3475,9 @@ mod tests { StubResponse::new(StatusCode::NO_CONTENT, ""), // remove stopped reader image_response("sha256:sandbox-runtime"), image_response("sha256:supervisor"), + StubResponse::new(StatusCode::NOT_FOUND, ""), // no existing private workspace StubResponse::new(StatusCode::CREATED, "{}"), // workspace volume + owned_volume_response(&container::volume_name(sandbox_id), sandbox_id), ]; if proxy_secret { responses.push(StubResponse::new(StatusCode::CREATED, "{}")); @@ -3188,10 +3488,26 @@ mod tests { sandbox_binary_archive_response(), StubResponse::new(StatusCode::NO_CONTENT, ""), // remove extractor ]); + responses.push(StubResponse::new(StatusCode::NOT_FOUND, "")); responses.push(StubResponse::new(StatusCode::CREATED, "{}")); // channel volume + responses.push(owned_volume_response( + &crate::isolation::channel_volume_name(sandbox_id), + sandbox_id, + )); responses } + fn owned_volume_response(name: &str, sandbox_id: &str) -> StubResponse { + StubResponse::new( + StatusCode::OK, + serde_json::json!({ + "Name": name, "Driver": "local", "Options": {}, + "Labels": {LABEL_SANDBOX_ID: sandbox_id, container::LABEL_SANDBOX_WORKSPACE: ""} + }) + .to_string(), + ) + } + fn create_launch_responses() -> Vec { vec![ created_response("workload"), @@ -3254,7 +3570,7 @@ mod tests { let auth_file = write_proxy_auth_file("create-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-container-fail", - create_setup_responses(true) + create_setup_responses(true, sandbox_id) .into_iter() .chain([ StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "create failed"), @@ -3297,7 +3613,7 @@ mod tests { let auth_file = write_proxy_auth_file("start-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-start-fail", - create_setup_responses(true) + create_setup_responses(true, sandbox_id) .into_iter() .chain(create_launch_responses().into_iter().take(7)) .chain([ diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 93a3f92966..d6ad260d0e 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -18,6 +18,13 @@ use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanC #[command(name = "openshell-driver-podman")] #[command(version = VERSION)] struct Args { + /// Operator-owned JSON policy; omitted means driver config disabled and labels required. + #[arg( + long, + env = "OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON", + default_value = "{}" + )] + admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig, /// Public compute-driver Unix socket used by an external gateway. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] bind_socket: Option, @@ -200,6 +207,8 @@ async fn main() -> Result<()> { ); let driver = PodmanComputeDriver::new(PodmanComputeConfig { + allow_driver_config: args.admission_config_json.allow_driver_config, + resource_admission: args.admission_config_json.resource_admission.clone(), socket_path: args.podman_socket, default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy, diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index f38fe901da..3f483577a8 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -8,6 +8,13 @@ The driver embeds libkrun, libkrunfw, the guest OCI unpacker, the portable guest ## How it fits together +Caller driver config is disabled by default, including upload workflows +encoded in driver JSON. GPU devices and their trusted VFIO plumbing are +temporarily exempt from approval labels; existing GPU selection validation +still applies. Private rootfs staging does not authorize arbitrary host paths. +The gateway passes its common policy to its managed VM subprocess. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). + ```mermaid flowchart LR subgraph host["Host"] diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 68f1de47c8..20a9c03bef 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -246,6 +246,12 @@ enum GuestImagePayloadSource { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct VmDriverConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + #[serde(default)] + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + #[serde(default)] + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, pub grpc_endpoint: String, pub state_dir: PathBuf, pub launcher_bin: Option, @@ -363,6 +369,9 @@ impl Default for VmDriverConfig { fn default() -> Self { Self { grpc_endpoint: String::new(), + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), state_dir: PathBuf::from("target/openshell-vm-driver"), launcher_bin: None, default_image: String::new(), @@ -671,6 +680,7 @@ impl VmDriver { mut config: VmDriverConfig, lifecycle_extensions: LifecycleExtensionRegistry, ) -> Result { + config.resource_admission.validate()?; lifecycle_extensions .validate() .map_err(|err| err.message().to_string())?; @@ -1010,6 +1020,11 @@ impl VmDriver { #[must_use] pub fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: DRIVER_NAME.to_string(), driver_version: openshell_core::VERSION.to_string(), default_image: self.config.default_image.clone(), @@ -1047,6 +1062,10 @@ impl VmDriver { // gRPC API surface; boxing here would diverge from every other handler. #[allow(clippy::result_large_err)] pub fn validate_sandbox(&self, sandbox: &Sandbox) -> Result<(), Status> { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + )?; validate_vm_sandbox(sandbox, self.config.gpu_enabled)?; let has_rootfs_tar = VmSandboxDriverConfig::from_sandbox(sandbox).is_ok_and(|c| c.rootfs_tar_path.is_some()); @@ -1062,6 +1081,7 @@ impl VmDriver { // gRPC API surface; boxing here would diverge from every other handler. #[allow(clippy::result_large_err)] pub async fn create_sandbox(&self, sandbox: &Sandbox) -> Result { + self.validate_sandbox(sandbox)?; info!( sandbox_id = %sandbox.id, sandbox_name = %sandbox.name, @@ -1850,6 +1870,14 @@ impl VmDriver { if launch_authentication.is_empty() { return Ok(()); } + } + let mut sandbox = read_sandbox_request(&state_dir.join(SANDBOX_REQUEST_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!("read VM admission provenance: {error}")) + })?; + self.validate_sandbox(&sandbox)?; + if already_running { // The gateway keeps launch sessions in memory. A non-empty bundle // during startup recovery represents a new gateway session, so // restart the VM before installing it rather than leaving the old @@ -1870,11 +1898,6 @@ impl VmDriver { ) .await .map_err(|error| Status::internal(format!("persist VM start generation: {error}")))?; - let mut sandbox = read_sandbox_request(&state_dir.join(SANDBOX_REQUEST_FILE)) - .await - .map_err(|err| { - Status::internal(format!("read sandbox start metadata failed: {err}")) - })?; let authentication = serde_json::from_slice::< openshell_core::jwt::SandboxLaunchAuthentication, >(&launch_authentication) @@ -2177,6 +2200,10 @@ impl VmDriver { clear_stop_marker: bool, reconciliation_span: &tracing::Span, ) -> bool { + if let Err(error) = self.validate_sandbox(&sandbox) { + warn!(sandbox_id = %sandbox.id, reason = %error.message(), "VM recovery denied by admission"); + return false; + } let has_rootfs_tar = VmSandboxDriverConfig::from_sandbox(&sandbox) .is_ok_and(|c| c.rootfs_tar_path.is_some()); @@ -8581,11 +8608,12 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); driver.config.state_dir = temp.path().to_path_buf(); + let (old_authentication, old_session) = test_launch_authentication("old"); let sandbox = Sandbox { id: "sandbox-stopped".to_string(), name: "stopped".to_string(), spec: Some(SandboxSpec { - launch_authentication: test_launch_authentication("old").0, + launch_authentication: old_authentication, ..Default::default() }), ..Default::default() @@ -8609,7 +8637,7 @@ mod tests { }, ); - let (fresh_authentication, fresh_session) = test_launch_authentication("fresh"); + let (fresh_authentication, _) = test_launch_authentication("fresh"); let err = driver .start_sandbox( &sandbox.id, @@ -8620,7 +8648,7 @@ mod tests { .await .expect_err("start without an image should fail"); - assert_eq!(err.code(), Code::Internal); + assert_eq!(err.code(), Code::FailedPrecondition); assert!( tokio::fs::metadata(state_dir.join(SANDBOX_STOPPED_FILE)) .await @@ -8650,10 +8678,54 @@ mod tests { .launch_authentication, ) .expect("persisted launch authentication"); - assert_eq!( - persisted_authentication.supervisor.session_id, - fresh_session + assert_eq!(persisted_authentication.supervisor.session_id, old_session); + } + + #[tokio::test] + async fn already_running_start_noop_does_not_require_admission_provenance() { + let temp = tempfile::tempdir().unwrap(); + let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + driver.config.state_dir = temp.path().to_path_buf(); + let sandbox = Sandbox { + id: "sandbox-running".to_string(), + name: "running".to_string(), + ..Default::default() + }; + let state_dir = temp.path().join("sandboxes").join(&sandbox.id); + create_private_dir_all(&state_dir).await.unwrap(); + tokio::fs::write( + state_dir.join(HOST_BOUNDARY_GENERATION_FILE), + b"g0000000000000001\n", + ) + .await + .unwrap(); + let provisioning_task = tokio::spawn(std::future::pending()); + let snapshot = sandbox_snapshot(&sandbox, provisioning_condition(), false); + driver.registry.lock().await.insert( + sandbox.id.clone(), + SandboxRecord { + snapshot, + state_dir, + process: None, + provisioning_task: Some(provisioning_task), + gpu_bdf: None, + deleting: false, + }, ); + + driver + .start_sandbox(&sandbox.id, &sandbox.name, "g0000000000000001", Vec::new()) + .await + .expect("matching already-running start must remain an idempotent no-op"); + + let task = driver + .registry + .lock() + .await + .remove(&sandbox.id) + .and_then(|record| record.provisioning_task) + .unwrap(); + task.abort(); } fn test_launch_authentication(label: &str) -> (Vec, openshell_core::SandboxSessionId) { diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 65f37eae56..eedb86fecb 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -25,6 +25,13 @@ use tracing::info; #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { + /// Operator-owned JSON policy; omitted means driver config disabled and labels required. + #[arg( + long, + env = "OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON", + default_value = "{}" + )] + admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig, #[arg(long, hide = true, default_value_t = false)] internal_run_vm: bool, @@ -274,6 +281,8 @@ async fn main() -> Result<()> { } let driver = VmDriver::new(VmDriverConfig { + allow_driver_config: args.admission_config_json.allow_driver_config, + resource_admission: args.admission_config_json.resource_admission.clone(), grpc_endpoint: args .grpc_endpoint .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index 32dcaeb24c..31e3975f0b 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -15,6 +15,7 @@ name = "openshell-gateway" path = "src/main.rs" [dependencies] +serde_json = { workspace = true } openshell-core = { path = "../openshell-core", default-features = false } openshell-server = { path = "../openshell-server", default-features = false } openshell-otel = { path = "../openshell-otel", optional = true } diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 86b7e37587..3b7bd6bf31 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -63,6 +63,8 @@ const COMPUTE_DRIVER_SOCKET_NAME: &str = "compute-driver.sock"; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] pub struct VmComputeConfig { + pub allow_driver_config: bool, + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// Working directory for VM driver sandbox state. pub state_dir: PathBuf, @@ -165,6 +167,7 @@ impl VmComputeConfig { /// Validate startup configuration without resolving binaries, creating /// state directories, spawning a process, or connecting a socket. pub fn validate_configuration(&self) -> Result<()> { + self.resource_admission.validate().map_err(Error::config)?; if self.grpc_endpoint.trim().is_empty() { return Err(Error::config( "grpc_endpoint is required when using the vm compute driver", @@ -232,6 +235,9 @@ impl Default for VmComputeConfig { fn default() -> Self { Self { state_dir: Self::default_state_dir(), + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), driver_dir: None, default_image: openshell_core::image::default_sandbox_image(), grpc_endpoint: String::new(), @@ -554,6 +560,13 @@ pub async fn spawn( command.stdout(Stdio::inherit()); command.stderr(Stdio::inherit()); command.arg("--bind-socket").arg(&socket_path); + command.arg("--admission-config-json").arg( + serde_json::to_string(&openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: vm_config.allow_driver_config, + resource_admission: vm_config.resource_admission.clone(), + }) + .map_err(|error| Error::config(error.to_string()))?, + ); command .arg("--expected-peer-pid") .arg(std::process::id().to_string()); diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index f264f4752f..6a2dfe44c2 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -114,6 +114,30 @@ pub struct DriverStartupContext<'a> { pub endpoint_overrides: &'a BTreeMap, } +/// Decode common controls without interpreting backend-specific driver fields. +pub fn admission_config_from_context( + context: DriverStartupContext<'_>, + name: &str, +) -> Result { + let mut table = toml::map::Map::new(); + if let Some(config) = context + .file + .and_then(|file| file.openshell.drivers.get(name)) + { + for field in ["allow_driver_config", "resource_admission"] { + if let Some(value) = config.get(field) { + table.insert(field.into(), value.clone()); + } + } + } + let policy: openshell_core::resource_admission::DriverAdmissionConfig = + toml::Value::Table(table).try_into().map_err(|error| { + Error::config(format!("invalid driver admission configuration: {error}")) + })?; + policy.validate().map_err(Error::config)?; + Ok(policy) +} + pub fn remote_driver_config_from_context( context: DriverStartupContext<'_>, name: &str, @@ -235,6 +259,37 @@ mod tests { } } + #[test] + fn common_admission_defaults_and_replacement_apply_to_every_driver() { + for name in ["kubernetes", "docker", "podman", "vm", "mxc", "external"] { + let defaults = admission_config_from_context(test_context(None), name).unwrap(); + assert!(!defaults.allow_driver_config); + assert!(defaults.resource_admission.enabled); + assert_eq!(defaults.resource_admission.required_labels.len(), 2); + let file: config_file::ConfigFile = toml::from_str(&format!( + "[openshell.drivers.{name}]\nallow_driver_config = true\n[openshell.drivers.{name}.resource_admission.required_labels]\n\"example.com/approved\" = \"yes\"\n" + )).unwrap(); + let custom = admission_config_from_context(test_context(Some(&file)), name).unwrap(); + assert!(custom.allow_driver_config); + assert_eq!( + custom.resource_admission.required_labels, + BTreeMap::from([("example.com/approved".into(), "yes".into())]) + ); + } + } + + #[test] + fn common_admission_rejects_empty_map_and_preserves_explicit_opt_out() { + let file: config_file::ConfigFile = + toml::from_str("[openshell.drivers.docker.resource_admission.required_labels]\n") + .unwrap(); + assert!(admission_config_from_context(test_context(Some(&file)), "docker").is_err()); + let file: config_file::ConfigFile = toml::from_str("[openshell.drivers.docker.resource_admission]\nenabled = false\nrequired_labels = {}\n").unwrap(); + let policy = admission_config_from_context(test_context(Some(&file)), "docker").unwrap(); + assert!(!policy.resource_admission.enabled); + assert!(!policy.allow_driver_config); + } + #[test] fn gateway_guest_tls_resolves_explicit_complete_bundle() { let dir = tempfile::tempdir().expect("temp dir"); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index ad98e46247..2a5ea8e4f2 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -455,6 +455,7 @@ impl AcquiredRemoteDriverEndpoint { #[derive(Debug, Clone)] struct RemoteComputeDriver { client: RemoteComputeDriverClient, + admission_acknowledgement: Arc>>, } type RemoteComputeDriverClient = ComputeDriverClient< @@ -465,12 +466,36 @@ impl RemoteComputeDriver { fn new(channel: Channel) -> Self { Self { client: ComputeDriverClient::with_interceptor(channel, TraceContextInterceptor), + admission_acknowledgement: Arc::new(StdMutex::new(None)), } } fn client(&self) -> RemoteComputeDriverClient { self.client.clone() } + + async fn verify_admission_policy(&self) -> Result<(), Status> { + let expected = self + .admission_acknowledgement + .lock() + .map_err(|_| Status::internal("admission handshake lock poisoned"))? + .clone() + .ok_or_else(|| Status::failed_precondition("driver admission handshake required"))?; + let current = self + .client() + .get_capabilities(GetCapabilitiesRequest { + gateway: Some(gateway_metadata(ExtensionFamily::Compute)), + }) + .await? + .into_inner() + .resource_admission_policy; + if current != expected { + return Err(Status::failed_precondition( + "remote driver admission policy changed; restart gateway after configuring matching policy", + )); + } + Ok(()) + } } #[tonic::async_trait] @@ -483,7 +508,12 @@ impl ComputeDriver for RemoteComputeDriver { ) -> Result, Status> { let mut client = self.client(); - client.get_capabilities(request).await + let response = client.get_capabilities(request).await?; + self.admission_acknowledgement + .lock() + .map_err(|_| Status::internal("admission handshake lock poisoned"))? + .get_or_insert_with(|| response.get_ref().resource_admission_policy.clone()); + Ok(response) } async fn authenticate_sandbox( @@ -505,6 +535,7 @@ impl ComputeDriver for RemoteComputeDriver { Status, > { let mut client = self.client(); + self.verify_admission_policy().await?; client.validate_sandbox_create(request).await } @@ -532,6 +563,7 @@ impl ComputeDriver for RemoteComputeDriver { ) -> Result, Status> { let mut client = self.client(); + self.verify_admission_policy().await?; client.create_sandbox(request).await } @@ -550,6 +582,7 @@ impl ComputeDriver for RemoteComputeDriver { ) -> Result, Status> { let mut client = self.client(); + self.verify_admission_policy().await?; client.start_sandbox(request).await } @@ -591,6 +624,8 @@ impl ComputeDriver for RemoteComputeDriver { #[derive(Clone)] pub struct ComputeRuntime { + admission: openshell_core::resource_admission::DriverAdmissionConfig, + admission_acknowledgement: String, driver: TracedDriver, driver_info: ComputeDriverInfoSnapshot, telemetry_compute_driver: TelemetryComputeDriver, @@ -688,6 +723,8 @@ impl ComputeRuntime { rootfs_tar_staging.sweep_orphans(); Ok(Self { driver: TracedDriver::new(driver, driver_name), + admission: openshell_core::resource_admission::DriverAdmissionConfig::default(), + admission_acknowledgement: capabilities.resource_admission_policy, driver_info, telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process, @@ -790,6 +827,33 @@ impl ComputeRuntime { &self.driver_info.name } + pub(crate) fn with_admission_policy( + mut self, + policy: openshell_core::resource_admission::DriverAdmissionConfig, + ) -> Result { + policy.verify_acknowledgement(&self.admission_acknowledgement)?; + if !policy.resource_admission.enabled { + warn!(driver = %self.driver_info.name, "External resource label admission is DISABLED"); + } + self.admission = policy; + Ok(self) + } + + pub(crate) fn validate_caller_driver_config( + &self, + template: Option<&SandboxTemplate>, + ) -> Result<(), Status> { + let selected = template + .map(|template| select_driver_config(&template.driver_config, &self.driver_info.name)) + .transpose() + .map_err(|error| *error)? + .flatten(); + openshell_core::resource_admission::check_driver_config( + self.admission.allow_driver_config, + selected.as_ref(), + ) + } + #[must_use] pub fn supports_sandbox_authentication(&self) -> bool { self.driver_info.supports_sandbox_authentication @@ -874,6 +938,12 @@ impl ComputeRuntime { } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { + self.validate_caller_driver_config( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + )?; let mut driver_sandbox = driver_sandbox_from_public(sandbox, &self.driver_info.name) .map_err(|status| *status)?; // Peek, never consume: create runs the same path immediately after and @@ -947,6 +1017,12 @@ impl ComputeRuntime { lifecycle_guard: SandboxLifecycleGuard, global_guard: SandboxSyncGuard, ) -> Result { + self.validate_caller_driver_config( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + )?; let sandbox_id = sandbox.object_id().to_string(); let mut sandbox = sandbox; @@ -1445,6 +1521,13 @@ impl ComputeRuntime { )); } + self.validate_caller_driver_config( + current + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + )?; + let mut attempts = 0; let (previous, starting, launch_authentication) = loop { let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); @@ -1474,7 +1557,6 @@ impl ComputeRuntime { "sandbox must be Stopped, Completed, or a failed main-process Error to start (current phase: {phase:?})" ))); } - if phase == SandboxPhase::Completed || is_failed_main_process_result(¤t) { self.cleanup_stopped_sandbox_sessions(¤t) .await @@ -1621,6 +1703,12 @@ impl ComputeRuntime { lifecycle_guard: SandboxLifecycleGuard, launch_authentication: Vec, ) -> Result { + self.validate_caller_driver_config( + starting + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + )?; let generation_id = sandbox_runtime_generation(&starting) .map_err(Status::failed_precondition)? .into_string(); @@ -2896,6 +2984,19 @@ impl ComputeRuntime { continue; } + if let Err(error) = self.validate_caller_driver_config( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + ) { + self.mark_sandbox_error(&sandbox, "ResourceAdmissionDenied", error.message()) + .await; + authentication_failed(sandbox.object_id()); + failed += 1; + continue; + } + let sandbox_name = sandbox.object_name().to_string(); let generation_id = match sandbox_runtime_generation(&sandbox) { Ok(generation) => generation.into_string(), @@ -3160,6 +3261,20 @@ impl ComputeRuntime { } } SandboxPhase::Starting => { + if let Err(error) = self.validate_caller_driver_config( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + ) { + self.mark_sandbox_error( + &sandbox, + "ResourceAdmissionDenied", + error.message(), + ) + .await; + continue; + } let sandbox_id = sandbox.object_id().to_string(); let sandbox_name = sandbox.object_name().to_string(); let driver_sandbox_id = sandbox_id.clone(); @@ -5886,6 +6001,9 @@ impl ComputeDriver for NoopTestDriver { Ok(tonic::Response::new( openshell_core::proto::compute::v1::GetCapabilitiesResponse { driver_name: "noop-test-driver".to_string(), + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, @@ -6061,6 +6179,11 @@ pub fn new_test_runtime_with_driver( let supports_sandbox_authentication = driver.sandbox_authentication.is_some(); ComputeRuntime { driver: TracedDriver::new(driver, "test".to_string()), + admission: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: true, + ..Default::default() + }, + admission_acknowledgement: String::new(), driver_info: ComputeDriverInfoSnapshot { name: driver_name.to_string(), driver_name: driver_name.to_string(), @@ -6477,6 +6600,9 @@ mod tests { ) -> Result, Status> { Ok(tonic::Response::new(GetCapabilitiesResponse { driver_name: "test-driver".to_string(), + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, @@ -6861,6 +6987,9 @@ mod tests { ) -> Result, Status> { Ok(tonic::Response::new(GetCapabilitiesResponse { driver_name: "controlled-test-driver".to_string(), + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, @@ -7097,6 +7226,11 @@ mod tests { let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); ComputeRuntime { driver: TracedDriver::new(driver, "test-driver".to_string()), + admission: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: true, + ..Default::default() + }, + admission_acknowledgement: String::new(), driver_info: ComputeDriverInfoSnapshot { name: driver_name.to_string(), driver_name: driver_name.to_string(), @@ -13278,7 +13412,7 @@ mod tests { let traceparents = driver.traceparents(); assert_eq!( traceparents.len(), - 8, + 10, "the client interceptor should cover every RPC" ); assert!( @@ -13349,7 +13483,7 @@ mod tests { .await .unwrap(); let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); - let runtime = ComputeRuntime::new_remote_driver( + let mut runtime = ComputeRuntime::new_remote_driver( endpoint, store, SandboxIndex::new(), @@ -13359,6 +13493,7 @@ mod tests { ) .await .unwrap(); + runtime.admission.allow_driver_config = true; let mut sandbox = sandbox_record("sb-uds", "uds-sandbox", SandboxPhase::Provisioning); sandbox.spec = Some(SandboxSpec { log_level: "debug".to_string(), @@ -13390,8 +13525,10 @@ mod tests { runtime.validate_sandbox_create(&sandbox).await.unwrap(); runtime.create_sandbox(sandbox, None, false).await.unwrap(); let calls = driver.calls(); - assert_eq!(calls.len(), 3, "unexpected calls: {calls:?}"); - let validated = match &calls[1] { + assert_eq!(calls.len(), 5, "unexpected calls: {calls:?}"); + assert!(matches!(calls[1], FakeComputeDriverCall::GetCapabilities)); + assert!(matches!(calls[3], FakeComputeDriverCall::GetCapabilities)); + let validated = match &calls[2] { FakeComputeDriverCall::ValidateSandboxCreate { sandbox: Some(sandbox), } => sandbox, @@ -13414,7 +13551,7 @@ mod tests { Some(42) ); assert!(matches!( - &calls[2], + &calls[4], FakeComputeDriverCall::CreateSandbox { sandbox: Some(sandbox) } if sandbox.spec.as_ref().and_then(|spec| spec.policy.as_ref()) .is_some_and(|policy| policy.version == 42) @@ -13435,7 +13572,7 @@ mod tests { runtime.start_persisted_sandboxes().await.unwrap(); assert!(matches!( driver.calls().as_slice(), - [FakeComputeDriverCall::StartSandbox { sandbox_id, sandbox_name }] + [FakeComputeDriverCall::GetCapabilities, FakeComputeDriverCall::StartSandbox { sandbox_id, sandbox_name }] if sandbox_id == "sb-uds" && sandbox_name == "uds-sandbox" )); driver.clear_calls(); @@ -13461,6 +13598,54 @@ mod tests { } } + #[tokio::test] + #[cfg(unix)] + async fn remote_compute_driver_rejects_policy_changes_before_forwarding() { + use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; + let dir = tempfile::tempdir().unwrap(); + let socket_path = dir.path().join("admission.sock"); + let driver = FakeComputeDriver::new(); + let _server = driver.serve_uds(&socket_path).unwrap(); + let endpoint = connect_remote_compute_driver("external-test", &socket_path) + .await + .unwrap(); + let remote = RemoteComputeDriver::new(endpoint.channel); + remote + .get_capabilities(Request::new(GetCapabilitiesRequest { + gateway: Some(gateway_metadata(ExtensionFamily::Compute)), + })) + .await + .unwrap(); + driver.set_admission_acknowledgement(String::new()); + driver.clear_calls(); + let sandbox = DriverSandbox::default(); + let status = remote + .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { + sandbox: Some(sandbox.clone()), + })) + .await + .unwrap_err(); + assert_eq!(status.code(), Code::FailedPrecondition); + assert!( + remote + .create_sandbox(Request::new(CreateSandboxRequest { + sandbox: Some(sandbox) + })) + .await + .is_err() + ); + assert!( + remote + .start_sandbox(Request::new(StartSandboxRequest::default())) + .await + .is_err() + ); + assert_eq!( + driver.calls(), + vec![FakeComputeDriverCall::GetCapabilities; 3] + ); + } + #[tokio::test] async fn create_sandbox_returns_resource_version_one() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index fd83e008e3..db8e7f017e 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -976,6 +976,10 @@ pub(super) async fn handle_create_sandbox_template( deletion_time: None, }); validate_sandbox_workload_template(&resolved)?; + let spec = sandbox_spec_from_user_workload_template(&resolved)?; + state + .compute + .validate_caller_driver_config(spec.template.as_ref())?; let labels_map = resolved.object_labels(); let labels_json = if labels_map.as_ref().is_none_or(HashMap::is_empty) { diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index c0a12facc7..4a8f2d5b35 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1664,6 +1664,8 @@ async fn build_compute_runtime( false, )?; let telemetry_compute_driver = driver.telemetry_compute_driver(registry); + let admission = + compute::driver_config::admission_config_from_context(driver_startup, driver.name())?; info!(driver = %driver.name(), "Using compute driver"); if config .gateway_jwt @@ -1745,6 +1747,9 @@ async fn build_compute_runtime( } }; + let runtime = runtime + .with_admission_policy(admission) + .map_err(Error::config)?; Ok(runtime.with_telemetry_compute_driver(telemetry_compute_driver)) } diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index a6b8d99e0b..dcbc2e1cf9 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -145,6 +145,9 @@ impl FakeComputeDriver { Self { state: Arc::new(Mutex::new(FakeComputeDriverState { capabilities: GetCapabilitiesResponse { + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), driver_name: "fake-compute-driver".to_string(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), @@ -206,6 +209,10 @@ impl FakeComputeDriver { self.with_state(|state| state.calls.clear()); } + pub fn set_admission_acknowledgement(&self, acknowledgement: String) { + self.with_state(|state| state.capabilities.resource_admission_policy = acknowledgement); + } + #[cfg(unix)] pub fn serve_uds( &self, diff --git a/deploy/helm/openshell-workspace/templates/role.yaml b/deploy/helm/openshell-workspace/templates/role.yaml index 45b3beb831..3df8928496 100644 --- a/deploy/helm/openshell-workspace/templates/role.yaml +++ b/deploy/helm/openshell-workspace/templates/role.yaml @@ -10,6 +10,10 @@ metadata: labels: {{- include "openshell-workspace.labels" . | nindent 4 }} rules: + # Metadata-only application admission still requires Kubernetes get permission. + - apiGroups: [""] + resources: ["persistentvolumeclaims", "configmaps", "secrets"] + verbs: ["get"] - apiGroups: - agents.x-k8s.io resources: diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 8fc1e4719f..70f0a56940 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -284,8 +284,11 @@ discovery endpoint or its TLS CA. | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | +| server.drivers.kubernetes.allowDriverConfig | bool | `false` | Allow caller driver JSON; external resources still require approval. | | server.drivers.kubernetes.operatorNamespaceFile | string | `""` | Path to a JSON file containing an array of namespace names allowed in operator mode. Hot-reloaded on change. | | server.drivers.kubernetes.operatorNamespaceLabel | string | `""` | K8s label selector for namespace discovery in operator mode. The driver watches namespaces matching this label. | +| server.drivers.kubernetes.resourceAdmission.enabled | bool | `true` | Require operator approval labels on external sandbox attachments (GPU attachments exempt). | +| server.drivers.kubernetes.resourceAdmission.requiredLabels | string | `nil` | Replacement label map; null uses the built-in admission labels. Empty map is invalid when enabled. | | server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 9265e0a96a..1df5c52467 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -9,6 +9,17 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get"] + - apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["get"] + {{- if ne $workspaceMode "shared" }} + - apiGroups: [""] + resources: ["persistentvolumeclaims", "configmaps", "secrets"] + verbs: ["get"] + {{- end }} # Validate projected ServiceAccount tokens during sandbox bootstrap and # internal gateway peer authentication. - apiGroups: diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index da3b8cec39..8c50b4053c 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -138,6 +138,7 @@ data: {{- end }} [openshell.drivers.kubernetes] + allow_driver_config = {{ .Values.server.drivers.kubernetes.allowDriverConfig }} namespace = {{ include "openshell.sandboxNamespace" . | quote }} default_image = {{ .Values.server.sandboxImage | quote }} {{- if include "openshell.sandboxRuntimeImageOverrideEnabled" . }} @@ -218,6 +219,15 @@ data: sandbox_runtime_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.sandboxRuntime.image.pullPolicy | quote }} {{- end }} + [openshell.drivers.kubernetes.resource_admission] + enabled = {{ .Values.server.drivers.kubernetes.resourceAdmission.enabled }} + {{- if ne .Values.server.drivers.kubernetes.resourceAdmission.requiredLabels nil }} + [openshell.drivers.kubernetes.resource_admission.required_labels] + {{- range $key, $value := .Values.server.drivers.kubernetes.resourceAdmission.requiredLabels }} + {{ $key | quote }} = {{ $value | quote }} + {{- end }} + {{- end }} + [openshell.drivers.kubernetes.managed_ssh_ingress] enabled = {{ .Values.networkPolicy.enabled }} gateway_namespace = {{ .Release.Namespace | quote }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index c781b46e3d..365ef3e9c7 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -10,6 +10,10 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: + # Metadata-only application admission still requires Kubernetes get permission. + - apiGroups: [""] + resources: ["persistentvolumeclaims", "configmaps", "secrets"] + verbs: ["get"] - apiGroups: - agents.x-k8s.io resources: diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 0a0428152e..9ee2e26ee5 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -13,6 +13,49 @@ release: namespace: my-namespace tests: + - it: defaults to label admission with caller driver config disabled + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^allow_driver_config\s*=\s*false$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.resource_admission\]\s*enabled\s*=\s*true' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '\[openshell\.drivers\.kubernetes\.resource_admission\.required_labels\]' + + - it: preserves explicit admission opt-out without enabling driver config + template: templates/gateway-config.yaml + set: + server.drivers.kubernetes.resourceAdmission.enabled: false + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^allow_driver_config\s*=\s*false$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.resource_admission\]\s*enabled\s*=\s*false' + + - it: renders replacement labels and literal workspace substitution + template: templates/gateway-config.yaml + set: + server.drivers.kubernetes.allowDriverConfig: true + server.drivers.kubernetes.resourceAdmission.requiredLabels: + platform.example.com/sandbox-attachable: "approved" + platform.example.com/team: '${workspace}' + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '"platform.example.com/sandbox-attachable" = "approved"' + - matchRegex: + path: data["gateway.toml"] + pattern: '"platform.example.com/team" = "\$\{workspace\}"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'openshell.ai/sandbox-attachable' + - it: identifies the gateway by chart fullname by default template: templates/gateway-config.yaml asserts: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 786038a8d4..84a59842a8 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -278,6 +278,13 @@ server: # Kubernetes compute driver settings. drivers: kubernetes: + # -- Allow caller driver JSON; external resources still require approval. + allowDriverConfig: false + resourceAdmission: + # -- Require operator approval labels on external sandbox attachments (GPU attachments exempt). + enabled: true + # -- Replacement label map; null uses the built-in admission labels. Empty map is invalid when enabled. + requiredLabels: null # -- How workspaces map to Kubernetes namespaces. # "shared" (default): all sandboxes in a single namespace. # "managed": auto-creates per-workspace namespaces. diff --git a/docs/kubernetes/sandbox-runtime.mdx b/docs/kubernetes/sandbox-runtime.mdx index aa039ad027..ffa7350f2e 100644 --- a/docs/kubernetes/sandbox-runtime.mdx +++ b/docs/kubernetes/sandbox-runtime.mdx @@ -87,6 +87,22 @@ cluster CNI enforces both ingress and egress policies for these namespaces. ## Bootstrap a Sandbox +Resource admission checks external references selected through the OpenShell-owned +Pod template before provisioning and again before releasing the workload's +scheduling gate. Kubernetes API-server and admission-webhook mutations of the +live Pod are trusted cluster-operator behavior and are not used as Workspace User +authorization inputs. PVCs and other supported external references require +operator approval labels, including same-namespace and read-only mounts. +Unexpected references, changed resource UIDs, and unsupported volume sources fail +closed. GPU device attachments are temporarily exempt. The driver rechecks +persisted references on restart and every 30 seconds during reconciliation; +confirmed revocation suspends compute, while transient lookup failures block new +launches and recovery. + +See [External Resource Admission](../reference/gateway-config#external-resource-admission) +for label configuration and upgrade requirements. Admission cannot repair data +or credentials compromised before upgrade. + The driver creates each sandbox generation in a fail-closed order: 1. Create and validate the workload egress fence. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index a5c322be89..a45da33fea 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -531,6 +531,129 @@ The gateway refuses to start when a peer endpoint is configured on a multi-repli ## Driver References +### External Resource Admission + +Every compute driver disables caller-supplied `template.driver_config` by +default. An operator can enable it without disabling resource approval: + +```toml +[openshell.drivers.kubernetes] +allow_driver_config = true + +[openshell.drivers.kubernetes.resource_admission] +enabled = true + +[openshell.drivers.kubernetes.resource_admission.required_labels] +"openshell.ai/sandbox-attachable" = "true" +"openshell.ai/sandbox-attachable-workspace" = "${workspace}" +``` + +Use the same fields under the selected Docker, Podman, VM, MXC, or external +driver table. Omitted admission settings use the labels above. A supplied map +replaces the defaults, and every entry must match. An explicitly empty map is +invalid while enforcement is enabled. `${workspace}` is the entire label value +and resolves to the authorized OpenShell workspace name, not a Kubernetes +namespace or caller label. Other substitutions and expressions are unsupported. +At least one fixed label is required so intentionally shared resources still +need explicit operator approval. + +The following label keys are reserved for driver-owned metadata and cannot be +used in `required_labels`: `openshell.ai/managed-by`, `openshell.ai/gateway-id`, +and `openshell.ai/sandbox-workspace`. The default admission key +`openshell.ai/sandbox-attachable-workspace` is distinct from the reserved +sandbox metadata key. + +The operator must label the referenced resource before use. Labels on a sandbox +do not approve its attachments. For example, a PVC used by workspace `team-a` +needs this metadata: + +```yaml +metadata: + labels: + openshell.ai/sandbox-attachable: "true" + openshell.ai/sandbox-attachable-workspace: "team-a" +``` + +Kubernetes checks PVCs, RuntimeClasses, PriorityClasses, and credential/config +references selected through the OpenShell-owned Pod template. Docker and Podman +check engine volume labels. Read-only mounts still require approval. Namespaces +and operator-selected defaults do not waive checks. Newly provisioned private +storage and bootstrap material use driver ownership checks instead. + +RuntimeClasses, PriorityClasses, and image-pull Secrets are shared operator +infrastructure. They must match every fixed required label, but `${workspace}` +entries do not apply to them. PVCs and Secret or ConfigMap data exposed to a +sandbox must also match the resolved workspace label. Kubernetes API-server and +admission-webhook mutations of the live Pod are trusted cluster-operator behavior +and are not used as Workspace User authorization inputs. + +With Kubernetes admission enabled, configured image-pull Secrets in the gateway +namespace must carry the fixed approval labels. In managed mode OpenShell verifies +that source, creates the workspace namespace, and copies the Secret with gateway +ownership metadata. An existing target Secret is updated only when its ownership +metadata matches the gateway and workspace. Operator mode still requires the +approved Secret in each operator-managed namespace. Automatic service-account +tokens and projected token volumes are prohibited. + +GPU device attachments are temporarily exempt from label admission. Existing +GPU validation still applies. Explicit GPU device settings inside driver JSON +still require `allow_driver_config = true`; public GPU-count requests do not. +The exception does not approve unrelated host mounts or non-GPU devices. + +Raw host bind mounts and supplemental image mounts have no trusted label +resolver and are rejected while admission is enabled. Publisher image labels +cannot grant operator approval. Docker's existing unsupported image-mount +restriction remains. MXC host filesystem grants likewise require disabling +admission until a resolver exists. MXC's command config and VM upload workflows +encoded in driver JSON require explicit driver-config opt-in. + +`allow_driver_config` applies to nonempty caller config, including saved +templates and non-storage overrides. An absent config or `{}` does not require +opt-in. Drivers recheck admitted OpenShell input attachments before launch and +on restart; running Docker, Podman, and Kubernetes workloads are periodically +revalidated. +Confirmed revocation stops or suspends the workload. Metadata lookup outages +block new launches but do not alone stop existing workloads. Revalidation uses +the Docker observation loop (normally two seconds) or a 30-second Podman / +Kubernetes interval, plus lookup and retry time. These are not instantaneous +revocation guarantees. + +To explicitly opt out of label authorization: + +```toml +[openshell.drivers.kubernetes.resource_admission] +enabled = false +``` + +This does not enable driver JSON or bypass existing path, ownership, namespace, +or isolation validation. It relinquishes external-resource label protection +for that driver. Restart the gateway and driver after policy changes. Only +trusted operators should control approval labels, runtime APIs, and resource +replacement; application admission is not atomic mount authorization. + +Before upgrading, inventory and approve legitimate external resources. Legacy +sandbox state without verifiable admission provenance requires recreation; +OpenShell does not automatically approve old attachments. Remove stale labels +before reusing a deleted workspace name. A custom approval-only map permits +cross-workspace sharing and should be an intentional operator decision. + +Standalone Kubernetes, Podman, and VM drivers accept the common policy as +`--admission-config-json` or `OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON`; `{}` uses +secure defaults. Docker also accepts the fields in its driver TOML file. The +gateway-managed VM process receives the selected gateway policy automatically. +Operator-managed external processes must be configured separately. Their +versioned `GetCapabilities.resource_admission_policy` acknowledgement must +match the gateway policy; legacy drivers require an explicit admission opt-out. +A changed acknowledgement blocks subsequent remote validation/create/start +calls until gateway and driver configuration agree again. + +Helm exposes `server.drivers.kubernetes.allowDriverConfig`, +`resourceAdmission.enabled`, and `resourceAdmission.requiredLabels`. The label +map defaults to `null`, which uses code defaults without merging those keys +into a custom map. Admission reads need Kubernetes `get` permissions even +though only metadata is requested. Never mark gateway database storage as +attachable. + Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Docker configurations use `sandbox_label`; the legacy `sandbox_namespace` key is rejected. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 2151c1a69b..8919dc554d 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -10,6 +10,14 @@ position: 4 The gateway's configured compute driver determines how OpenShell creates each sandbox. The CLI workflow stays the same across drivers: you create, connect to, inspect, stop, start, and delete sandboxes through the gateway API. +Caller driver config is disabled by default across drivers. Enabling it does +not authorize external attachments: referenced resources must carry matching +operator-controlled approval labels. GPU device attachments are temporarily +exempt. Host bind mounts, supplemental image mounts, and MXC host filesystem +grants lack a trusted label resolver and are unavailable with admission enabled. +See [External Resource Admission](gateway-config#external-resource-admission) +for configuration, migration, and the explicit unsafe opt-out. + Most compute drivers run the OpenShell supervisor inside the sandbox workload. The supervisor launches the agent process, applies policy, routes egress through the proxy, injects configured credentials, and maintains the gateway session. @@ -199,7 +207,8 @@ For GPU-backed Docker sandboxes, configure Docker CDI before starting the gatewa Docker driver config accepts user-supplied `volume` and `tmpfs` mounts. It also accepts `bind` mounts when `[openshell.drivers.docker]` sets -`enable_bind_mounts = true` in `gateway.toml`. See Docker's [storage documentation](https://docs.docker.com/engine/storage/) for more information. +`enable_bind_mounts = true` and explicitly disables resource admission in +`gateway.toml`. All examples require `allow_driver_config = true`. See Docker's [storage documentation](https://docs.docker.com/engine/storage/) for more information. Docker local-driver named volumes created with bind options also expose gateway-host paths, so OpenShell treats them like bind mounts and requires `enable_bind_mounts = true`. @@ -207,7 +216,8 @@ gateway-host paths, so OpenShell treats them like bind mounts and requires Use a `volume` mount for existing Docker named volumes: ```shell -docker volume create openshell-work +docker volume create --label openshell.ai/sandbox-attachable=true \ + --label openshell.ai/sandbox-attachable-workspace=default openshell-work openshell sandbox create \ --driver-config-json '{"docker":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work","read_only":false}]}}' \ @@ -221,11 +231,16 @@ workspace isolation and filesystem policy. Use them only when you understand and accept that loss of isolation. -Use a `bind` mount only after enabling it in the Docker driver table: +Raw paths cannot satisfy label admission. The following unsafe opt-out permits +bind mounts and removes label protection for all Docker attachments: ```toml [openshell.drivers.docker] +allow_driver_config = true enable_bind_mounts = true + +[openshell.drivers.docker.resource_admission] +enabled = false ``` ```shell @@ -281,9 +296,11 @@ your Podman machine uses a non-standard host-loopback address, or set ### Podman Driver Config Mounts -Podman driver config accepts user-supplied `volume`, `tmpfs`, and `image` -mounts. It also accepts `bind` mounts when `[openshell.drivers.podman]` sets -`enable_bind_mounts = true` in `gateway.toml`. Podman local-driver named +Podman driver config accepts user-supplied `volume` and `tmpfs` mounts. +All examples require `allow_driver_config = true`. Supplemental `image` mounts +require disabling resource admission. It also accepts `bind` mounts when +`[openshell.drivers.podman]` sets `enable_bind_mounts = true` and explicitly +disables resource admission in `gateway.toml`. Podman local-driver named volumes created with bind options also expose gateway-host paths, so OpenShell treats them like bind mounts and requires `enable_bind_mounts = true`. Host bind mounts expose gateway host paths to sandbox requests, so they are @@ -292,7 +309,8 @@ disabled by default. Use a `volume` mount for existing Podman named volumes: ```shell -podman volume create openshell-work +podman volume create --label openshell.ai/sandbox-attachable=true \ + --label openshell.ai/sandbox-attachable-workspace=default openshell-work openshell sandbox create \ --driver-config-json '{"podman":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work","read_only":false}]}}' \ @@ -306,11 +324,16 @@ workspace isolation and filesystem policy. Use them only when you understand and accept that loss of isolation. -Use a `bind` mount only after enabling it in the Podman driver table: +Raw paths cannot satisfy label admission. The following unsafe opt-out permits +bind mounts and removes label protection for all Podman attachments: ```toml [openshell.drivers.podman] +allow_driver_config = true enable_bind_mounts = true + +[openshell.drivers.podman.resource_admission] +enabled = false ``` ```shell diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index ad4ad65410..f1343f1cfb 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -59,6 +59,7 @@ source "${ROOT}/e2e/support/podman-gateway-config.sh" mkdir -p "${WORKDIR}/pki/client" "${WORKDIR}/jwt" e2e_write_podman_gateway_config "${WORKDIR}/v1.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 '' e2e_write_podman_gateway_config "${WORKDIR}/v2.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 '' +e2e_write_podman_gateway_config "${WORKDIR}/v2-external.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 1 socket network 18181 image:test 15 supervisor:test '' '' 0 '' assert_contains "${WORKDIR}/v1.toml" 'version = 1' assert_contains "${WORKDIR}/v1.toml" 'compute_drivers = ["podman"]' assert_contains "${WORKDIR}/v1.toml" 'image_pull_policy = "missing"' @@ -67,7 +68,12 @@ assert_contains "${WORKDIR}/v1.toml" 'guest_tls_ca = ' assert_contains "${WORKDIR}/v2.toml" 'version = 2' assert_contains "${WORKDIR}/v2.toml" 'compute_driver = "podman"' assert_contains "${WORKDIR}/v2.toml" 'image_pull_policy = "if_not_present"' +assert_contains "${WORKDIR}/v2.toml" 'allow_driver_config = true' +assert_contains "${WORKDIR}/v2.toml" '[openshell.drivers.podman.resource_admission]' assert_not_contains "${WORKDIR}/v2.toml" 'health_check_interval_secs = 0' +assert_contains "${WORKDIR}/v2-external.toml" 'socket_path = "socket"' +assert_not_contains "${WORKDIR}/v2-external.toml" 'allow_driver_config = true' +assert_not_contains "${WORKDIR}/v2-external.toml" '[openshell.drivers.podman.resource_admission]' # V2 guest TLS is emitted before its driver table; V1 is driver-local. OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v1-options.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 "" OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v2-options.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 "" diff --git a/e2e/support/podman-gateway-config.sh b/e2e/support/podman-gateway-config.sh index 1f604ebcf7..9d6ea929e1 100755 --- a/e2e/support/podman-gateway-config.sh +++ b/e2e/support/podman-gateway-config.sh @@ -151,6 +151,7 @@ e2e_write_podman_gateway_config() { if [ "${external_driver}" = "1" ]; then printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${driver_socket}")" else + printf 'allow_driver_config = true\n' printf 'network_name = %s\n' "$(e2e_podman_toml_string "${network_name}")" printf 'gateway_port = %s\n' "${gateway_port}" printf 'default_image = %s\n' "$(e2e_podman_toml_string "${sandbox_image}")" @@ -168,6 +169,8 @@ e2e_write_podman_gateway_config() { if [ -n "${podman_socket}" ]; then printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${podman_socket}")" fi + printf '\n[openshell.drivers.podman.resource_admission]\n' + printf 'enabled = false\n' fi e2e_write_gateway_jwt_config "${jwt_dir}" "${gateway_id}" if [ "${oidc_mode}" != "1" ]; then diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index e4d0d8128f..c4b5a446ed 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -599,6 +599,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else + printf 'allow_driver_config = true\n' printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" @@ -606,6 +607,8 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'enable_bind_mounts = true\n' printf 'sandbox_runtime_image = %s\n' "$(toml_string "${SANDBOX_RUNTIME_IMAGE}")" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" + printf '\n[openshell.drivers.docker.resource_admission]\n' + printf 'enabled = false\n' fi } > "${GATEWAY_CONFIG}" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index b9c195b050..8c6659d638 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1378,6 +1378,9 @@ else --docker-server=registry.example.test \ --docker-username=e2e-user \ --docker-password=e2e-password + kctl -n "${NAMESPACE}" label secret \ + "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET}" \ + openshell.ai/sandbox-attachable=true fi if [ "${OPENSHIFT_DETECTED}" = "1" ]; then diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index d7875d67e6..f278174b5c 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -102,6 +102,10 @@ message GetCapabilitiesResponse { uint64 rootfs_tar_max_bytes = 11; // Compute extension protocol metadata. Required for protocol negotiation. openshell.extension.v1.PeerMetadata extension = 12; + // Versioned effective operator admission policy (v1: followed by JSON). + // Gateways require an exact policy match before activating the driver. + // Empty denotes a legacy driver and requires explicit admission opt-out. + string resource_admission_policy = 13; } message AuthenticateSandboxRequest { diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 258c1ebde7..420ae24e8d 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -92,6 +92,22 @@ Use gateway metadata, deployment values, or the user's setup notes to identify t Before debugging the compute platform, inspect gateway logs for failures in dependencies initialized before the listener becomes ready. +For resource-admission failures, distinguish disabled caller driver config from +missing resource approval. Helm defaults `server.drivers.kubernetes.allowDriverConfig` +to false and `resourceAdmission.enabled` to true. Existing PVCs, Secrets, +ConfigMaps, RuntimeClasses, and PriorityClasses need matching administrator-owned +labels; namespace membership and read-only access do not grant approval. Inspect +metadata only when diagnosing credentials. GPU devices do not need labels. +RuntimeClasses, PriorityClasses, and image-pull Secrets match fixed approval +labels but do not use the workspace placeholder. In managed mode, inspect the +approved source image-pull Secret in the gateway namespace and the gateway-owned +copy in the workspace namespace. Legacy workloads without admission provenance need recreation. Do not +automatically label control-plane resources or disable enforcement as a repair. + +For out-of-tree compute drivers, also check that their versioned admission-policy +acknowledgement matches the gateway's policy. Configure standalone driver policy +through its administrator-owned `--admission-config-json` option. + For out-of-tree compute drivers, confirm the selected driver name and socket agree across CLI flags or `gateway.toml`, and that the operator-owned driver is running before the gateway starts: ```bash diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index b6711e5140..89c707d424 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -301,8 +301,15 @@ openshell sandbox template create gpu-kata \ openshell sandbox create --name my-sandbox --template gpu-kata --provider my-github -- claude ``` -Direct `sandbox create --driver-config-json` remains valid for one-off -creates. Put driver config on a template only when it should be reused. +Driver config is disabled by default. These template and one-off +`sandbox create --driver-config-json` examples require the administrator to set +`allow_driver_config = true` for the selected driver. This does not waive +resource admission: external attachments need administrator-controlled approval +labels on the actual resources, not sandbox labels. GPU device attachments +are temporarily exempt from labels; the public `--gpu` flag needs no driver +config opt-in. Consult the published gateway configuration reference before +changing admission settings; do not recommend disabling admission to bypass a +denial. Put driver config on a template only when it should be reused. ### Manage sandbox workload templates From bdffa102c389f62243828b740c9334e0b498d6f0 Mon Sep 17 00:00:00 2001 From: Mrunal Patel Date: Tue, 22 Sep 2026 21:52:19 +0000 Subject: [PATCH 5/9] feat(api): add durable exec launch admission (#3324) * feat(api): add durable exec launch admission Fence duplicate exec launches with keyed durable admission and producer-owned terminal completion. Keep uncertain launches unresolved and never replay output or interactive input. Part of #3051 (phase 4a). Signed-off-by: Mrunal Patel * fix(api): fence exec identity across authorization lookups Signed-off-by: Mrunal Patel --------- Signed-off-by: Mrunal Patel --- architecture/gateway.md | 20 +- crates/openshell-cli/src/run.rs | 2 + crates/openshell-sdk/src/client.rs | 2 + crates/openshell-server/src/grpc/mod.rs | 2 +- .../src/grpc/mutation_replay.rs | 129 +++- .../src/grpc/mutation_replay/streaming.rs | 148 ++++ .../grpc/mutation_replay/streaming/tests.rs | 720 ++++++++++++++++++ .../src/grpc/mutation_replay/tests.rs | 1 + crates/openshell-server/src/grpc/sandbox.rs | 195 +++-- crates/openshell-server/src/storage_proto.rs | 13 +- docs/reference/api-errors.mdx | 46 +- e2e/python/test_exec_admission.py | 157 ++++ proto/openshell.proto | 6 + sdk/go/proto/openshellv1/openshell.pb.go | 20 +- 14 files changed, 1372 insertions(+), 89 deletions(-) create mode 100644 crates/openshell-server/src/grpc/mutation_replay/streaming.rs create mode 100644 crates/openshell-server/src/grpc/mutation_replay/streaming/tests.rs create mode 100644 e2e/python/test_exec_admission.py diff --git a/architecture/gateway.md b/architecture/gateway.md index 747d328921..413c126552 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -95,7 +95,25 @@ in-memory extension. Replay reauthorizes original and current effective scopes, requires the same effective payload, and reruns current interceptor validation. Interceptors cannot mutate the request UUID. Server-marked replay suppresses post-commit observation, which remains best-effort rather than an outbox. -Credential capabilities and streaming execution require separate contracts. +Credential capabilities require separate contracts. + +Both exec RPCs share keyed admission but defer completion to the SSH producer. +The initial interactive Start uses the exec request schema under a distinct RPC +namespace; later stdin and resize frames are not replayable inputs. Admission +resolves the public sandbox name and workspace, durably binds both original and +effective sandbox UUIDs, and rejects same-name replacements. When the original +and effective selectors match, both authorization lookups must resolve the same +workspace and sandbox identities before admission. A different effective target +is allowed only when the selector changes. The owner checks the effective UUID +again before relay opening, then hands its CAS-only finalizer to the owned producer, +releasing the shared admission-worker permit after handoff. Only a confirmed +remote exit records a terminal marker and starts 24-hour retention. Synthetic +timeouts, disconnects without exit confirmation, and persistence failures leave +permanent unresolved claims. Duplicates never launch, attach, or replay output: +pending records report uncertainty, and terminal records report stream +unavailability. Existing transport cancellation behavior remains unchanged. +Exec timeouts retain the public duration's precision and presence: an absent +timeout is unbounded, while an explicit zero is a finite timeout. The gateway listens on one service port and multiplexes gRPC and HTTP traffic. The default local single-user deployment mode is mTLS user authentication: diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index b6292f5dc9..74d92351ff 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1897,6 +1897,7 @@ pub async fn sandbox_exec_grpc( // Make the streaming gRPC call. let mut stream = client .exec_sandbox(ExecSandboxRequest { + request_id: String::new(), sandbox: name.to_string(), workspace_scope: Some(openshell_core::proto::workspace_selector( workspace.to_string(), @@ -2300,6 +2301,7 @@ async fn sandbox_exec_interactive_grpc( input_tx .send(ExecSandboxInput { payload: Some(exec_sandbox_input::Payload::Start(ExecSandboxRequest { + request_id: String::new(), sandbox: sandbox.object_name().to_string(), workspace_scope: Some(openshell_core::proto::workspace_selector( (sandbox.object_workspace()).to_string(), diff --git a/crates/openshell-sdk/src/client.rs b/crates/openshell-sdk/src/client.rs index 59b42e7997..0bf2412282 100644 --- a/crates/openshell-sdk/src/client.rs +++ b/crates/openshell-sdk/src/client.rs @@ -607,6 +607,7 @@ impl OpenShellClient { /// call `exec_sandbox` directly. pub async fn exec(&self, name: &str, cmd: &[String], opts: ExecOptions) -> Result { let request = proto::ExecSandboxRequest { + request_id: String::new(), sandbox: name.to_string(), workspace_scope: Some(proto::workspace_selector("default")), command: cmd.to_vec(), @@ -1185,6 +1186,7 @@ impl WorkspaceScopedClient { /// Run a command inside a sandbox and buffer stdout/stderr. pub async fn exec(&self, name: &str, cmd: &[String], opts: ExecOptions) -> Result { let request = proto::ExecSandboxRequest { + request_id: String::new(), sandbox: name.to_string(), workspace_scope: Some(proto::workspace_selector(&self.workspace)), command: cmd.to_vec(), diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index 9a6f65198d..5ac58307a4 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -430,7 +430,7 @@ impl OpenShell for OpenShellService { &self, request: Request, ) -> Result, Status> { - sandbox::handle_exec_sandbox(&self.state, request).await + mutation_replay::run(&self.state, request).await } type ForwardTcpStream = diff --git a/crates/openshell-server/src/grpc/mutation_replay.rs b/crates/openshell-server/src/grpc/mutation_replay.rs index a6f8267041..19ce936888 100644 --- a/crates/openshell-server/src/grpc/mutation_replay.rs +++ b/crates/openshell-server/src/grpc/mutation_replay.rs @@ -56,6 +56,8 @@ struct Admission { #[serde(default)] protection: Option, workspace_id: Option, + #[serde(default)] + target_id: Option, success: Option, completed_at_ms: Option, } @@ -65,6 +67,8 @@ struct Protection { key_id: String, effective_payload_hash: String, effective_workspace_id: Option, + #[serde(default)] + effective_target_id: Option, } /// Gateway-owned, in-memory input before hydration and interceptor modification. @@ -78,19 +82,39 @@ pub(super) enum Success { Resource { id: String, version: u64 }, Deletion { outcome: i32 }, Ordinary(ordinary::Outcome), + StreamTerminal, } pub(super) struct Scope { name: String, workspace_id: Option, + // Name-addressed exec requests must remain bound to one sandbox instance. + target_id: Option, +} + +#[cfg(test)] +#[derive(Default)] +struct AuthorizationBarrier { + resolved: tokio::sync::Notify, + resume: tokio::sync::Notify, } #[tonic::async_trait] pub(super) trait Mutation: Message + Default + Send + Sync + 'static { - type Output: Message + Default + Send + 'static; + type Output: Send + 'static; const METHOD: &'static str; const PROTECTED: bool = false; + // Streaming producers, not returned stream handles, confirm completion. + const DEFERRED: bool = false; fn request_id(&self) -> &str; + /// Name-addressed targets must keep their identity unless an interceptor + /// actually changes the selector, not merely another request field. + fn target_selector(&self) -> Option<(&str, &WorkspaceSelector)> { + None + } + fn canonical_message(&self) -> Result { + decode_request_message(&format!("{}Request", Self::METHOD), self) + } async fn authorize(&self, state: &ServerState, principal: &Principal) -> Result; async fn execute( state: &Arc, @@ -132,7 +156,19 @@ pub(super) async fn run( )); }; let scope = original.authorize(state, &principal).await?; + #[cfg(test)] + if let Some(barrier) = request.extensions().get::>() { + barrier.resolved.notify_one(); + barrier.resume.notified().await; + } let effective_scope = request.get_ref().authorize(state, &principal).await?; + if original.target_selector().is_some() + && original.target_selector() == request.get_ref().target_selector() + && (scope.workspace_id != effective_scope.workspace_id + || scope.target_id != effective_scope.target_id) + { + return Err(replay_unavailable()); + } let (payload_hash, protection) = if M::PROTECTED { let key = fingerprint_key(state).await?; ( @@ -141,6 +177,7 @@ pub(super) async fn run( key_id: key.id(), effective_payload_hash: key.fingerprint(request.get_ref())?, effective_workspace_id: effective_scope.workspace_id, + effective_target_id: effective_scope.target_id, }), ) } else { @@ -207,11 +244,12 @@ async fn execute_owned( protection: Option, scope: Scope, ) -> Result, Status> { - let mut admission = Admission { + let admission = Admission { format_version: 1, payload_hash, protection, workspace_id: scope.workspace_id, + target_id: scope.target_id, success: None, completed_at_ms: None, }; @@ -256,6 +294,7 @@ async fn execute_owned( )); } if previous.workspace_id != admission.workspace_id + || previous.target_id != admission.target_id || previous.protection != admission.protection { return Err(replay_unavailable()); @@ -302,32 +341,85 @@ async fn execute_owned( }; // Any error or panic from here leaves the claim unresolved. Status codes // do not establish that a handler performed no effects. + let completion = Completion { + store: state.store.clone(), + claim_id, + key: key.into(), + bucket: bucket.into(), + version, + admission: payload, + }; + if M::DEFERRED { + request.extensions_mut().insert(completion.clone()); + } let facts = ordinary::Facts::default(); request.extensions_mut().insert(facts.clone()); let mut response = M::execute(state, request).await?; + if M::DEFERRED { + return Ok(response); + } response.extensions_mut().insert(facts); - admission.success = Some(M::capture(&response)?); + completion.complete(M::capture(&response)?).await?; + return Ok(response); + } + Err(uncertain()) +} + +/// Only the admitted owner can finalize this exact incarnation of a claim. +/// A dropped finalizer leaves a permanent unresolved admission, never a lease. +#[derive(Clone)] +pub(super) struct Completion { + store: Arc, + claim_id: String, + key: String, + bucket: String, + version: u64, + admission: Vec, +} + +impl Completion { + /// Reject replacement between admission authorization and relay creation. + pub(super) fn ensure_target(&self, target_id: &str) -> Result<(), Status> { + let admission: Admission = + serde_json::from_slice(&self.admission).map_err(|_| uncertain())?; + if admission + .protection + .as_ref() + .and_then(|protection| protection.effective_target_id.as_deref()) + != Some(target_id) + { + return Err(uncertain()); + } + Ok(()) + } + + pub(super) async fn stream_terminal(&self) -> Result<(), Status> { + self.complete(Success::StreamTerminal).await + } + + async fn complete(&self, success: Success) -> Result<(), Status> { + let mut admission: Admission = + serde_json::from_slice(&self.admission).map_err(|_| uncertain())?; + admission.success = Some(success); admission.completed_at_ms = Some(current_time_ms()); let payload = serde_json::to_vec(&admission).map_err(|_| uncertain())?; if payload.len() > 64 * 1024 { return Err(uncertain()); } - state - .store + self.store .put_if( OBJECT_TYPE, - &claim_id, - key, - bucket, + &self.claim_id, + &self.key, + &self.bucket, &payload, None, - WriteCondition::MatchResourceVersion(version), + WriteCondition::MatchResourceVersion(self.version), ) .await .map_err(|_| uncertain())?; - return Ok(response); + Ok(()) } - Err(uncertain()) } async fn prune_expired(store: &Store, bucket: &str) -> Result { @@ -377,12 +469,16 @@ fn validate_request_id(value: &str) -> Result { Ok(id.hyphenated().to_string()) } -fn fingerprint(request: &M) -> Result { +fn decode_request_message(name: &str, request: &impl Message) -> Result { let descriptor = DESCRIPTORS - .get_message_by_name(&format!("openshell.v1.{}Request", M::METHOD)) + .get_message_by_name(&format!("openshell.v1.{name}")) .ok_or_else(|| Status::internal("mutation request descriptor missing"))?; - let mut message = DynamicMessage::decode(descriptor, request.encode_to_vec().as_slice()) - .map_err(|_| Status::internal("decode mutation request"))?; + DynamicMessage::decode(descriptor, request.encode_to_vec().as_slice()) + .map_err(|_| Status::internal("decode mutation request")) +} + +fn fingerprint(request: &M) -> Result { + let mut message = request.canonical_message()?; message.clear_field_by_name("request_id"); let value = serde_json::to_value(message) .map_err(|_| Status::internal("canonicalize mutation request"))?; @@ -492,6 +588,7 @@ async fn named_scope(state: &ServerState, name: &str) -> Result { Ok(Scope { name: name.into(), workspace_id: Some(resource.object_id().into()), + target_id: None, }) } @@ -500,6 +597,7 @@ fn global_scope(state: &ServerState, principal: &Principal) -> Result>; + +/// Private, one-owner transport handoff. The fingerprint includes only Start. +#[derive(Clone)] +pub(in crate::grpc) struct InteractiveInput( + pub Arc>>>, +); + +pub(in crate::grpc) fn start(input: &ExecSandboxInput) -> Result<&ExecSandboxRequest, Status> { + match input.payload.as_ref() { + Some(exec_sandbox_input::Payload::Start(request)) => Ok(request), + _ => Err(rpc_error::invalid_argument( + "start", + "first message must be a start payload", + )), + } +} + +async fn authorize( + state: &ServerState, + principal: &Principal, + request: &ExecSandboxRequest, +) -> Result { + sandbox::validate_exec_start(request)?; + let sandbox = sandbox::resolve_and_authorize_sandbox_name( + state, + principal, + &request.sandbox, + crate::auth::workspace_authz::selected_workspace_name(request.workspace_scope.as_ref())?, + crate::auth::workspace_authz::MinWorkspaceRole::User, + ) + .await?; + let mut scope = named_scope(state, sandbox.object_workspace()).await?; + scope.target_id = Some(sandbox.object_id().into()); + Ok(scope) +} + +fn stream_unavailable(success: Success) -> Status { + if !matches!(success, Success::StreamTerminal) { + return super::replay_unavailable(); + } + rpc_error::failed_precondition( + "REQUEST_STREAM_UNAVAILABLE", + "execution terminated, but its output stream is not stored; this request was not launched again", + ) +} + +#[tonic::async_trait] +impl Mutation for ExecSandboxRequest { + type Output = ExecStream; + const METHOD: &'static str = "ExecSandbox"; + const PROTECTED: bool = true; + const DEFERRED: bool = true; + + fn request_id(&self) -> &str { + &self.request_id + } + + fn target_selector(&self) -> Option<(&str, &WorkspaceSelector)> { + self.workspace_scope + .as_ref() + .map(|workspace| (self.sandbox.as_str(), workspace)) + } + + async fn authorize(&self, state: &ServerState, principal: &Principal) -> Result { + authorize(state, principal, self).await + } + + async fn execute( + state: &Arc, + request: Request, + ) -> Result, Status> { + sandbox::handle_exec_sandbox(state, request).await + } + + fn capture(_: &Response) -> Result { + Err(uncertain()) + } + + async fn restore(_: &Store, success: Success) -> Result { + Err(stream_unavailable(success)) + } +} + +#[tonic::async_trait] +impl Mutation for ExecSandboxInput { + type Output = ExecStream; + const METHOD: &'static str = "ExecSandboxInteractive"; + const PROTECTED: bool = true; + const DEFERRED: bool = true; + + fn request_id(&self) -> &str { + start(self).map_or("", |request| request.request_id.as_str()) + } + + fn target_selector(&self) -> Option<(&str, &WorkspaceSelector)> { + start(self).ok().and_then(Mutation::target_selector) + } + + fn canonical_message(&self) -> Result { + decode_request_message("ExecSandboxRequest", start(self)?) + } + + async fn authorize(&self, state: &ServerState, principal: &Principal) -> Result { + authorize(state, principal, start(self)?).await + } + + async fn execute( + state: &Arc, + request: Request, + ) -> Result, Status> { + sandbox::handle_exec_sandbox_interactive_start(state, request).await + } + + fn capture(_: &Response) -> Result { + Err(uncertain()) + } + + async fn restore(_: &Store, success: Success) -> Result { + Err(stream_unavailable(success)) + } +} diff --git a/crates/openshell-server/src/grpc/mutation_replay/streaming/tests.rs b/crates/openshell-server/src/grpc/mutation_replay/streaming/tests.rs new file mode 100644 index 0000000000..112cf64157 --- /dev/null +++ b/crates/openshell-server/src/grpc/mutation_replay/streaming/tests.rs @@ -0,0 +1,720 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::grpc::mutation_replay::tests::{reason, state_for}; +use crate::grpc::mutation_replay::{ + Admission, AuthorizationBarrier, Completion, OBJECT_TYPE, OriginalMutation, SUCCESS_TTL_MS, + fingerprint, run, +}; +use crate::grpc::test_support::authed_request; +use crate::persistence::{ObjectType, WriteCondition, current_time_ms}; +use openshell_core::proto::datamodel::v1::ObjectMeta; +use openshell_core::proto::{ + GatewayMessage, Sandbox, SandboxPhase, SandboxStatus, gateway_message, +}; +use openshell_core::{ObjectId, ObjectName}; +use prost::Message; +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::StreamExt; + +type ExecClient = + openshell_core::proto::open_shell_client::OpenShellClient; + +async fn wire_client( + state: Arc, + barrier: Option>, + original: Option, +) -> (ExecClient, tokio::task::JoinHandle<()>) { + use crate::grpc::OpenShellService; + use openshell_core::proto::open_shell_server::OpenShellServer; + use tokio_stream::wrappers::TcpListenerStream; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let principal = authed_request(()) + .extensions() + .get::() + .unwrap() + .clone(); + let service = OpenShellServer::with_interceptor( + OpenShellService::new(state), + move |mut request: Request<()>| { + request.extensions_mut().insert(principal.clone()); + if let Some(barrier) = &barrier { + request.extensions_mut().insert(barrier.clone()); + } + if let Some(original) = &original { + request.extensions_mut().insert(original.clone()); + } + Ok(request) + }, + ); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .unwrap(); + }); + let client = ExecClient::connect(format!("http://{address}")) + .await + .unwrap(); + (client, server) +} + +fn interactive_start(req: ExecSandboxRequest) -> ExecSandboxInput { + ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Start(req)), + } +} + +fn original_exec(req: ExecSandboxRequest, interactive: bool) -> OriginalMutation { + OriginalMutation(if interactive { + interactive_start(req).encode_to_vec() + } else { + req.encode_to_vec() + }) +} + +async fn exec_rpc( + client: &mut ExecClient, + req: ExecSandboxRequest, + interactive: bool, +) -> Result>, Status> { + if interactive { + client + .exec_sandbox_interactive(tokio_stream::iter([interactive_start(req)])) + .await + } else { + client.exec_sandbox(req).await + } +} + +async fn setup(url: &str, directory: &tempfile::TempDir) -> Arc { + let mut state = state_for(Store::connect(url).await.unwrap()).await; + let key = directory.path().join("key"); + std::fs::write(&key, b"exec-test-only-key").unwrap(); + let state_mut = Arc::get_mut(&mut state).unwrap(); + state_mut.admin_role = "openshell-admin".into(); + state_mut.config.gateway_jwt = Some(openshell_core::config::GatewayJwtConfig { + signing_key_path: key, + public_key_path: directory.path().join("public"), + kid_path: directory.path().join("kid"), + gateway_id: "test".into(), + ttl_secs: None, + }); + state +} + +async fn request(state: &ServerState) -> ExecSandboxRequest { + let sandbox = Sandbox { + metadata: Some(ObjectMeta { + id: uuid::Uuid::new_v4().to_string(), + name: "exec-admission".into(), + workspace: "default".into(), + ..Default::default() + }), + status: Some(SandboxStatus { + phase: SandboxPhase::Ready.into(), + ..Default::default() + }), + ..Default::default() + }; + state.store.put_message(&sandbox).await.unwrap(); + ExecSandboxRequest { + sandbox: sandbox.object_name().into(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + request_id: uuid::Uuid::new_v4().to_string(), + command: vec!["echo".into(), "secret-command-argument".into()], + stdin: b"secret-stdin".to_vec(), + environment: [("PRIVATE".into(), "secret-environment".into())].into(), + ..Default::default() + } +} + +async fn sandbox_id(state: &ServerState, req: &ExecSandboxRequest) -> String { + state + .store + .get_message_by_name::("default", &req.sandbox) + .await + .unwrap() + .unwrap() + .object_id() + .into() +} + +fn register(state: &ServerState, id: &str) -> mpsc::Receiver { + let (tx, rx) = mpsc::channel(64); + let (shutdown, _) = oneshot::channel(); + state + .supervisor_sessions + .register(id.into(), "session".into(), tx, shutdown); + rx +} + +async fn receive_relay(state: &ServerState, messages: &mut mpsc::Receiver) { + let message = tokio::time::timeout(std::time::Duration::from_secs(2), messages.recv()) + .await + .unwrap() + .unwrap(); + let Some(gateway_message::Payload::RelayOpen(open)) = message.payload else { + panic!("expected relay open") + }; + // No SSH transport in these admission tests. Release the detached producer. + state + .supervisor_sessions + .fail_pending_relay(&open.channel_id, "test relay ended".into()); +} + +async fn completion(state: &ServerState) -> Completion { + let rows = state + .store + .list_by_type_after(OBJECT_TYPE, None, 100) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + let row = rows.into_iter().next().unwrap(); + Completion { + store: state.store.clone(), + claim_id: row.id, + key: row.name, + bucket: row.workspace, + version: row.resource_version, + admission: row.payload, + } +} + +#[test] +fn interactive_fingerprint_uses_start_schema_and_ignores_request_id() { + let mut req = ExecSandboxRequest { + request_id: uuid::Uuid::new_v4().to_string(), + command: vec!["echo".into()], + ..Default::default() + }; + let first = fingerprint(&req).unwrap(); + req.request_id = uuid::Uuid::new_v4().to_string(); + let input = ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Start(req.clone())), + }; + assert_eq!(first, fingerprint(&input).unwrap()); + assert_ne!(ExecSandboxRequest::METHOD, ExecSandboxInput::METHOD); + req.stdin.push(42); + assert_ne!(first, fingerprint(&req).unwrap()); + assert_eq!( + ExecSandboxInput::default() + .canonical_message() + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); +} + +#[tokio::test] +async fn concurrent_exec_claims_open_one_relay_and_store_no_secrets() { + let directory = tempfile::tempdir().unwrap(); + let state = setup("sqlite::memory:", &directory).await; + let req = request(&state).await; + let mut messages = register(&state, &sandbox_id(&state, &req).await); + let mut tasks = Vec::new(); + for _ in 0..16 { + let state = state.clone(); + let req = req.clone(); + tasks.push(tokio::spawn(async move { + run(&state, authed_request(req)).await + })); + } + let mut admitted = 0; + for task in tasks { + match task.await.unwrap() { + Ok(_) => admitted += 1, + Err(status) => assert_eq!(reason(&status), "REQUEST_OUTCOME_UNCERTAIN"), + } + } + assert_eq!(admitted, 1); + receive_relay(&state, &mut messages).await; + assert!(messages.try_recv().is_err()); + let pending = completion(&state).await; + let text = String::from_utf8(pending.admission).unwrap(); + for secret in [ + "secret-command-argument", + "secret-stdin", + "secret-environment", + &fingerprint(&req).unwrap(), + ] { + assert!(!text.contains(secret)); + } + let mut changed = req.clone(); + changed.command.push("different".into()); + assert_eq!( + reason(&run(&state, authed_request(changed)).await.unwrap_err()), + "REQUEST_ID_PAYLOAD_MISMATCH" + ); +} + +#[tokio::test] +async fn confirmed_exit_including_nonzero_and_124_finalizes_before_output_delivery() { + for code in [0, 7, 124] { + let directory = tempfile::tempdir().unwrap(); + let state = setup("sqlite::memory:", &directory).await; + let req = request(&state).await; + let mut messages = register(&state, &sandbox_id(&state, &req).await); + drop(run(&state, authed_request(req.clone())).await.unwrap()); + receive_relay(&state, &mut messages).await; + let owner = completion(&state).await; + assert_eq!( + sandbox::wait_for_exec_terminal(async { Ok(code) }, None, Some(owner.clone())) + .await + .unwrap(), + Some(code) + ); + assert_eq!( + reason(&run(&state, authed_request(req)).await.unwrap_err()), + "REQUEST_STREAM_UNAVAILABLE" + ); + assert!( + owner.stream_terminal().await.is_err(), + "CAS rejects a second finalization" + ); + let row = completion(&state).await; + let admission: Admission = serde_json::from_slice(&row.admission).unwrap(); + assert!(matches!(admission.success, Some(Success::StreamTerminal))); + assert!(admission.completed_at_ms.is_some()); + } +} + +#[tokio::test] +async fn timeout_and_lost_exit_status_never_finalize_or_expire_pending_launches() { + let directory = tempfile::tempdir().unwrap(); + let state = setup("sqlite::memory:", &directory).await; + let req = request(&state).await; + let mut messages = register(&state, &sandbox_id(&state, &req).await); + drop(run(&state, authed_request(req.clone())).await.unwrap()); + receive_relay(&state, &mut messages).await; + let owner = completion(&state).await; + for execution_timeout in [ + std::time::Duration::ZERO, + std::time::Duration::from_nanos(1), + std::time::Duration::from_secs(1), + ] { + assert_eq!( + tokio::time::timeout( + std::time::Duration::from_secs(5), + sandbox::wait_for_exec_terminal( + std::future::pending(), + Some(execution_timeout), + Some(owner.clone()), + ), + ) + .await + .expect("a present timeout must not wait indefinitely") + .unwrap(), + None + ); + } + assert!( + sandbox::wait_for_exec_terminal( + async { Err(Status::unavailable("lost exit")) }, + None, + Some(owner.clone()) + ) + .await + .is_err() + ); + let mut admission: Admission = serde_json::from_slice(&owner.admission).unwrap(); + assert!(admission.success.is_none()); + admission.completed_at_ms = Some(current_time_ms() - SUCCESS_TTL_MS - 1); + state + .store + .put_if( + OBJECT_TYPE, + &owner.claim_id, + &owner.key, + &owner.bucket, + &serde_json::to_vec(&admission).unwrap(), + None, + WriteCondition::MatchResourceVersion(owner.version), + ) + .await + .unwrap(); + // A real exit cannot be exposed as confirmed if the finalizer loses its CAS. + let status = sandbox::wait_for_exec_terminal(async { Ok(0) }, None, Some(owner)) + .await + .unwrap_err(); + assert_eq!(reason(&status), "REQUEST_OUTCOME_UNCERTAIN"); + assert_eq!( + reason(&run(&state, authed_request(req)).await.unwrap_err()), + "REQUEST_OUTCOME_UNCERTAIN" + ); + assert!(messages.try_recv().is_err()); +} + +#[tokio::test] +async fn cancellation_before_admission_does_nothing_and_after_admission_keeps_owner() { + let directory = tempfile::tempdir().unwrap(); + let state = setup("sqlite::memory:", &directory).await; + let req = request(&state).await; + drop(run(&state, authed_request(req.clone()))); + assert!( + state + .store + .list_by_type_after(OBJECT_TYPE, None, 100) + .await + .unwrap() + .is_empty() + ); + let task_state = state.clone(); + let task_req = req.clone(); + let task = tokio::spawn(async move { run(&task_state, authed_request(task_req)).await }); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while state + .store + .list_by_type_after(OBJECT_TYPE, None, 100) + .await + .unwrap() + .is_empty() + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + task.abort(); + let mut messages = register(&state, &sandbox_id(&state, &req).await); + receive_relay(&state, &mut messages).await; + assert_eq!( + reason(&run(&state, authed_request(req)).await.unwrap_err()), + "REQUEST_OUTCOME_UNCERTAIN" + ); + assert!(messages.try_recv().is_err()); +} + +#[tokio::test] +async fn exec_restart_preserves_pending_and_terminal_fences_and_reauthorizes() { + let directory = tempfile::tempdir().unwrap(); + let url = format!( + "sqlite://{}?mode=rwc", + directory.path().join("store.db").display() + ); + let state = setup(&url, &directory).await; + let req = request(&state).await; + let mut messages = register(&state, &sandbox_id(&state, &req).await); + let mut output = run(&state, authed_request(req.clone())) + .await + .unwrap() + .into_inner(); + receive_relay(&state, &mut messages).await; + assert!(output.next().await.unwrap().is_err()); + assert!(output.next().await.is_none()); + drop(output); + drop(messages); + drop(state); + let state = setup(&url, &directory).await; + assert_eq!( + reason(&run(&state, authed_request(req.clone())).await.unwrap_err()), + "REQUEST_OUTCOME_UNCERTAIN" + ); + completion(&state).await.stream_terminal().await.unwrap(); + drop(state); + let state = setup(&url, &directory).await; + let mut sandbox: Sandbox = state + .store + .get_message_by_name("default", &req.sandbox) + .await + .unwrap() + .unwrap(); + sandbox.status.as_mut().unwrap().phase = SandboxPhase::Stopped.into(); + state.store.put_message(&sandbox).await.unwrap(); + assert_eq!( + reason(&run(&state, authed_request(req.clone())).await.unwrap_err()), + "REQUEST_STREAM_UNAVAILABLE" + ); + let mut unauthorized = authed_request(req.clone()); + if let Principal::User(user) = unauthorized + .extensions_mut() + .get_mut::() + .unwrap() + { + user.identity.roles.clear(); + } + assert_eq!( + run(&state, unauthorized).await.unwrap_err().code(), + tonic::Code::NotFound + ); + state + .store + .delete(Sandbox::object_type(), sandbox.object_id()) + .await + .unwrap(); + assert_eq!( + run(&state, authed_request(req)).await.unwrap_err().code(), + tonic::Code::NotFound + ); +} + +#[tokio::test] +async fn noninteractive_rejects_replacement_between_authorizations() { + replacement_between_authorizations(false).await; +} + +#[tokio::test] +async fn interactive_rejects_replacement_between_authorizations() { + replacement_between_authorizations(true).await; +} + +async fn replacement_between_authorizations(interactive: bool) { + // A payload-only transformation must not be mistaken for redirection. + for change_command in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let state = setup("sqlite::memory:", &directory).await; + let mut req = request(&state).await; + let original = change_command.then(|| original_exec(req.clone(), interactive)); + if change_command { + req.command.push("transformed-argument".into()); + } + let id = sandbox_id(&state, &req).await; + let mut old_messages = register(&state, &id); + let barrier = Arc::new(AuthorizationBarrier::default()); + let (mut client, server) = + wire_client(state.clone(), Some(barrier.clone()), original).await; + let call = tokio::spawn(async move { exec_rpc(&mut client, req, interactive).await }); + tokio::time::timeout( + std::time::Duration::from_secs(5), + barrier.resolved.notified(), + ) + .await + .unwrap(); + + let mut replacement: Sandbox = state.store.get_message(&id).await.unwrap().unwrap(); + state + .store + .delete(Sandbox::object_type(), &id) + .await + .unwrap(); + replacement.metadata.as_mut().unwrap().id = uuid::Uuid::new_v4().to_string(); + state.store.put_message(&replacement).await.unwrap(); + let mut replacement_messages = register(&state, replacement.object_id()); + barrier.resume.notify_one(); + + let status = tokio::time::timeout(std::time::Duration::from_secs(5), call) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(reason(&status), "REQUEST_REPLAY_UNAVAILABLE"); + assert!(old_messages.try_recv().is_err()); + assert!(replacement_messages.try_recv().is_err()); + assert!( + state + .store + .list_by_type_after(OBJECT_TYPE, None, 100) + .await + .unwrap() + .is_empty() + ); + server.abort(); + } +} + +#[tokio::test] +async fn exec_transformations_preserve_unchanged_targets_and_explicit_redirection() { + use openshell_core::proto::Workspace; + + for interactive in [false, true] { + for redirect in ["command-only", "sandbox", "workspace"] { + let directory = tempfile::tempdir().unwrap(); + let state = setup("sqlite::memory:", &directory).await; + let mut req = request(&state).await; + let original = original_exec(req.clone(), interactive); + let original_id = sandbox_id(&state, &req).await; + let mut target: Sandbox = state + .store + .get_message(&original_id) + .await + .unwrap() + .unwrap(); + req.command.push("transformed-argument".into()); + if redirect != "command-only" { + target.metadata.as_mut().unwrap().id = uuid::Uuid::new_v4().to_string(); + if redirect == "sandbox" { + req.sandbox = "redirected".into(); + target.metadata.as_mut().unwrap().name = req.sandbox.clone(); + } else { + let workspace = Workspace { + metadata: Some(ObjectMeta { + id: uuid::Uuid::new_v4().to_string(), + name: "redirected".into(), + ..Default::default() + }), + ..Default::default() + }; + state.store.put_message(&workspace).await.unwrap(); + target.metadata.as_mut().unwrap().workspace = "redirected".into(); + req.workspace_scope = + Some(openshell_core::proto::workspace_selector("redirected")); + } + state.store.put_message(&target).await.unwrap(); + } + let mut messages = register(&state, target.object_id()); + let (mut client, server) = wire_client(state.clone(), None, Some(original)).await; + drop(exec_rpc(&mut client, req, interactive).await.unwrap()); + receive_relay(&state, &mut messages).await; + let owner = completion(&state).await; + let admission: Admission = serde_json::from_slice(&owner.admission).unwrap(); + assert_eq!(admission.target_id.as_deref(), Some(original_id.as_str())); + owner.ensure_target(target.object_id()).unwrap(); + if redirect != "command-only" { + assert!(owner.ensure_target(&original_id).is_err()); + } + server.abort(); + } + } +} + +#[tokio::test] +async fn replacement_cannot_reuse_an_exec_claim_or_receive_its_admitted_launch() { + for terminal in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let state = setup("sqlite::memory:", &directory).await; + let req = request(&state).await; + let id = sandbox_id(&state, &req).await; + let mut messages = register(&state, &id); + drop(run(&state, authed_request(req.clone())).await.unwrap()); + receive_relay(&state, &mut messages).await; + let owner = completion(&state).await; + owner.ensure_target(&id).unwrap(); + if terminal { + owner.stream_terminal().await.unwrap(); + } + let mut replacement: Sandbox = state.store.get_message(&id).await.unwrap().unwrap(); + state + .store + .delete(Sandbox::object_type(), &id) + .await + .unwrap(); + replacement.metadata.as_mut().unwrap().id = uuid::Uuid::new_v4().to_string(); + state.store.put_message(&replacement).await.unwrap(); + let mut replacement_messages = register(&state, replacement.object_id()); + + assert_eq!( + reason(&run(&state, authed_request(req.clone())).await.unwrap_err()), + "REQUEST_REPLAY_UNAVAILABLE" + ); + // Simulate replacement after admission, before the owner resolves its + // public name again. The original claim must not launch on the new ID. + let mut admitted = authed_request(req); + admitted.extensions_mut().insert(owner); + assert_eq!( + reason( + &sandbox::handle_exec_sandbox(&state, admitted) + .await + .unwrap_err() + ), + "REQUEST_OUTCOME_UNCERTAIN" + ); + assert!(replacement_messages.try_recv().is_err()); + } +} + +#[tokio::test] +async fn interactive_wire_start_is_admitted_under_its_own_method_namespace() { + use crate::grpc::OpenShellService; + use openshell_core::proto::{ + open_shell_client::OpenShellClient, open_shell_server::OpenShellServer, + }; + use tokio_stream::wrappers::TcpListenerStream; + let directory = tempfile::tempdir().unwrap(); + let state = setup("sqlite::memory:", &directory).await; + let req = request(&state).await; + let mut messages = register(&state, &sandbox_id(&state, &req).await); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let principal = authed_request(()) + .extensions() + .get::() + .unwrap() + .clone(); + let service = OpenShellServer::with_interceptor( + OpenShellService::new(state.clone()), + move |mut request: Request<()>| { + request.extensions_mut().insert(principal.clone()); + Ok(request) + }, + ); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .unwrap(); + }); + let mut client = OpenShellClient::connect(format!("http://{address}")) + .await + .unwrap(); + let start = ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Start(req.clone())), + }; + let output = client + .exec_sandbox_interactive(tokio_stream::iter([start.clone()])) + .await + .unwrap(); + receive_relay(&state, &mut messages).await; + let owner = completion(&state).await; + assert_eq!( + reason( + &client + .exec_sandbox_interactive(tokio_stream::iter([start.clone()])) + .await + .unwrap_err() + ), + "REQUEST_OUTCOME_UNCERTAIN" + ); + owner.stream_terminal().await.unwrap(); + assert_eq!( + reason( + &client + .exec_sandbox_interactive(tokio_stream::iter([start.clone()])) + .await + .unwrap_err() + ), + "REQUEST_STREAM_UNAVAILABLE" + ); + // The same UUID belongs to a different RPC namespace, not another stdin stream. + drop(client.exec_sandbox(req.clone()).await.unwrap()); + receive_relay(&state, &mut messages).await; + assert_eq!( + state + .store + .list_by_type_after(OBJECT_TYPE, None, 100) + .await + .unwrap() + .len(), + 2 + ); + let id = sandbox_id(&state, &req).await; + let mut replacement: Sandbox = state.store.get_message(&id).await.unwrap().unwrap(); + state + .store + .delete(Sandbox::object_type(), &id) + .await + .unwrap(); + replacement.metadata.as_mut().unwrap().id = uuid::Uuid::new_v4().to_string(); + state.store.put_message(&replacement).await.unwrap(); + let mut replacement_messages = register(&state, replacement.object_id()); + assert_eq!( + reason( + &client + .exec_sandbox_interactive(tokio_stream::iter([start])) + .await + .unwrap_err() + ), + "REQUEST_REPLAY_UNAVAILABLE" + ); + assert_eq!( + reason(&client.exec_sandbox(req).await.unwrap_err()), + "REQUEST_REPLAY_UNAVAILABLE" + ); + assert!(replacement_messages.try_recv().is_err()); + drop(output); + server.abort(); +} diff --git a/crates/openshell-server/src/grpc/mutation_replay/tests.rs b/crates/openshell-server/src/grpc/mutation_replay/tests.rs index fa1a54d7b0..28dfc688f2 100644 --- a/crates/openshell-server/src/grpc/mutation_replay/tests.rs +++ b/crates/openshell-server/src/grpc/mutation_replay/tests.rs @@ -733,6 +733,7 @@ async fn quota_fails_closed_but_replays_and_expired_success_cleanup_still_work() format_version: 1, payload_hash: "unused".into(), workspace_id: None, + target_id: None, success: None, completed_at_ms: None, }; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index db8e7f017e..1428b766db 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -161,7 +161,7 @@ impl Drop for WatchSandboxStream { /// Missing and unauthorized objects deliberately share one response so names /// cannot be used as an existence oracle. pub(super) async fn resolve_and_authorize_sandbox_name( - state: &Arc, + state: &ServerState, principal: &crate::auth::principal::Principal, sandbox_name: &str, workspace: &str, @@ -2341,16 +2341,12 @@ pub(super) async fn handle_exec_sandbox( use openshell_core::ObjectId; let principal = super::extract_principal(&request)?; + let completion = request + .extensions() + .get::() + .cloned(); let req = request.into_inner(); - if req.command.is_empty() { - return Err(Status::invalid_argument("command is required")); - } - if req.environment.keys().any(|key| !is_valid_env_key(key)) { - return Err(Status::invalid_argument( - "environment keys must match ^[A-Za-z_][A-Za-z0-9_]*$", - )); - } - validate_exec_request_fields(&req)?; + validate_exec_start(&req)?; let sandbox = resolve_and_authorize_sandbox_name( state, @@ -2361,6 +2357,10 @@ pub(super) async fn handle_exec_sandbox( ) .await?; + if let Some(completion) = &completion { + completion.ensure_target(sandbox.object_id())?; + } + if SandboxPhase::try_from(sandbox.phase()).ok() != Some(SandboxPhase::Ready) { return Err(Status::failed_precondition("sandbox is not ready")); } @@ -2415,6 +2415,7 @@ pub(super) async fn handle_exec_sandbox( no_login_shell, cols, rows, + completion, ) .await { @@ -2767,6 +2768,29 @@ async fn bridge_forward_tcp_stream( // Interactive exec handler (bidirectional stdin streaming) // --------------------------------------------------------------------------- +pub(super) fn validate_exec_start(req: &ExecSandboxRequest) -> Result<(), Status> { + use openshell_core::rpc_error; + if req.sandbox.is_empty() { + return Err(rpc_error::invalid_argument( + "sandbox", + "sandbox is required", + )); + } + if req.command.is_empty() { + return Err(rpc_error::invalid_argument( + "command", + "command is required", + )); + } + if req.environment.keys().any(|key| !is_valid_env_key(key)) { + return Err(rpc_error::invalid_argument( + "environment", + "environment keys must match ^[A-Za-z_][A-Za-z0-9_]*$", + )); + } + validate_exec_request_fields(req) +} + fn validate_interactive_exec_start( msg: Option, ) -> Result { @@ -2781,15 +2805,7 @@ fn validate_interactive_exec_start( )); }; - if req.command.is_empty() { - return Err(Status::invalid_argument("command is required")); - } - if req.environment.keys().any(|key| !is_valid_env_key(key)) { - return Err(Status::invalid_argument( - "environment keys must match ^[A-Za-z_][A-Za-z0-9_]*$", - )); - } - validate_exec_request_fields(&req)?; + validate_exec_start(&req)?; Ok(req) } @@ -2798,10 +2814,7 @@ pub(super) async fn handle_exec_sandbox_interactive( state: &Arc, request: Request>, ) -> Result>>, Status> { - use openshell_core::ObjectId; - - let principal = super::extract_principal(&request)?; - let mut input_stream = request.into_inner(); + let (metadata, extensions, mut input_stream) = request.into_parts(); let first_msg = input_stream .message() @@ -2810,6 +2823,46 @@ pub(super) async fn handle_exec_sandbox_interactive( let req = validate_interactive_exec_start(first_msg)?; + let mut request = Request::from_parts( + metadata, + extensions, + ExecSandboxInput { + payload: Some(openshell_core::proto::exec_sandbox_input::Payload::Start( + req, + )), + }, + ); + request + .extensions_mut() + .insert(super::mutation_replay::streaming::InteractiveInput( + Arc::new(tokio::sync::Mutex::new(Some(input_stream))), + )); + super::mutation_replay::run(state, request).await +} + +pub(super) async fn handle_exec_sandbox_interactive_start( + state: &Arc, + request: Request, +) -> Result>>, Status> { + let principal = super::extract_principal(&request)?; + let completion = request + .extensions() + .get::() + .cloned(); + let input = request + .extensions() + .get::() + .ok_or_else(|| Status::internal("interactive input handoff missing"))? + .clone(); + let input_stream = input + .0 + .lock() + .await + .take() + .ok_or_else(|| Status::internal("interactive input already claimed"))?; + let req = super::mutation_replay::streaming::start(request.get_ref())?; + validate_exec_start(req)?; + let sandbox = resolve_and_authorize_sandbox_name( state, &principal, @@ -2819,6 +2872,10 @@ pub(super) async fn handle_exec_sandbox_interactive( ) .await?; + if let Some(completion) = &completion { + completion.ensure_target(sandbox.object_id())?; + } + if SandboxPhase::try_from(sandbox.phase()).ok() != Some(SandboxPhase::Ready) { return Err(Status::failed_precondition("sandbox is not ready")); } @@ -2833,7 +2890,7 @@ pub(super) async fn handle_exec_sandbox_interactive( .await .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; - let command_str = build_remote_exec_command(&req) + let command_str = build_remote_exec_command(req) .map_err(|e| Status::invalid_argument(format!("command construction failed: {e}")))?; let request_tty = req.tty; let no_login_shell = req.no_login_shell; @@ -2873,6 +2930,7 @@ pub(super) async fn handle_exec_sandbox_interactive( execution_timeout, cols, rows, + completion, ) .await { @@ -3184,6 +3242,7 @@ async fn stream_exec_over_relay( no_login_shell: bool, cols: u32, rows: u32, + completion: Option, ) -> Result<(), Status> { let command_preview: String = command .chars() @@ -3213,26 +3272,22 @@ async fn stream_exec_over_relay( tx.clone(), ); - let exec_result = if let Some(execution_timeout) = execution_timeout { - if let Ok(result) = tokio::time::timeout(execution_timeout, exec).await { - result - } else { - let _ = tx - .send(Ok(ExecSandboxEvent { - payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Exit( - ExecSandboxExit { exit_code: 124 }, - )), - })) - .await; - finish_interactive_exec_proxy(proxy_task).await; - return Ok(()); - } - } else { - exec.await - }; + let exec_result = wait_for_exec_terminal(exec, execution_timeout, completion).await; + if matches!(exec_result, Ok(None)) { + let _ = tx + .send(Ok(ExecSandboxEvent { + payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Exit( + ExecSandboxExit { exit_code: 124 }, + )), + })) + .await; + finish_interactive_exec_proxy(proxy_task).await; + return Ok(()); + } let exit_code = match exec_result { - Ok(code) => code, + Ok(Some(code)) => code, + Ok(None) => unreachable!("timeout returned above"), Err(status) => { finish_interactive_exec_proxy(proxy_task).await; return Err(status); @@ -3265,6 +3320,7 @@ async fn stream_interactive_exec_over_relay( execution_timeout: Option, cols: u32, rows: u32, + completion: Option, ) -> Result<(), Status> { let command_preview: String = command .chars() @@ -3294,26 +3350,22 @@ async fn stream_interactive_exec_over_relay( tx.clone(), ); - let exec_result = if let Some(execution_timeout) = execution_timeout { - if let Ok(result) = tokio::time::timeout(execution_timeout, exec).await { - result - } else { - let _ = tx - .send(Ok(ExecSandboxEvent { - payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Exit( - ExecSandboxExit { exit_code: 124 }, - )), - })) - .await; - finish_interactive_exec_proxy(proxy_task).await; - return Ok(()); - } - } else { - exec.await - }; + let exec_result = wait_for_exec_terminal(exec, execution_timeout, completion).await; + if matches!(exec_result, Ok(None)) { + let _ = tx + .send(Ok(ExecSandboxEvent { + payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Exit( + ExecSandboxExit { exit_code: 124 }, + )), + })) + .await; + finish_interactive_exec_proxy(proxy_task).await; + return Ok(()); + } let exit_code = match exec_result { - Ok(code) => code, + Ok(Some(code)) => code, + Ok(None) => unreachable!("timeout returned above"), Err(status) => { finish_interactive_exec_proxy(proxy_task).await; return Err(status); @@ -3333,6 +3385,29 @@ async fn stream_interactive_exec_over_relay( Ok(()) } +/// `Ok(Some(code))` requires a real SSH `ExitStatus`, including nonzero codes. +/// Synthetic gateway timeouts, lost status, and dropped futures leave the claim +/// unresolved. Finalization precedes terminal delivery to the client. +pub(super) async fn wait_for_exec_terminal( + exec: impl Future>, + execution_timeout: Option, + completion: Option, +) -> Result, Status> { + let result = if let Some(execution_timeout) = execution_timeout { + match tokio::time::timeout(execution_timeout, exec).await { + Ok(result) => result, + Err(_) => return Ok(None), + } + } else { + exec.await + }; + let exit_code = result?; + if let Some(completion) = completion { + completion.stream_terminal().await?; + } + Ok(Some(exit_code)) +} + async fn finish_interactive_exec_proxy(mut task: tokio::task::JoinHandle<()>) { if tokio::time::timeout(EXEC_POST_EXIT_CLOSE_TIMEOUT, &mut task) .await diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 77fe7937a0..196b7be1cd 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -118,15 +118,12 @@ mod tests { const STORAGE_V1_SCHEMA_SHA256: &str = "d68401809d8cea445c35233ef32412bbd041cb2ac5acaf368a0d0bf74d2ddf17"; - // Carries both this branch's opaque watch cursor and the well-known time - // types from main. The cursor change adds `WatchSandboxRequest` - // `.resume_after_cursor` and `SandboxStreamEvent.cursor` as strings; both - // fields are unreleased, added on this branch, so no client depends on an - // earlier type. It adds no messages or enums and touches no stored type, - // which is why the durable and overlap fingerprints below are main's - // values unchanged. + // Carries this branch's exec request IDs together with main's opaque watch + // cursor and well-known time types. These unreleased public-only fields add + // no messages or enums and touch no stored type, so the durable and overlap + // fingerprints below remain unchanged. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "fd8a5cad432441be1fe8891332b56f52335c208cc7283dcd93e385e7ae66c6da"; + "e9533b91b8ead17666c43b17ca5bca3bf84050346e4855e5e5a261cbb56a26bb"; const DURABLE_SCHEMA_SHA256: &str = "9eeaa29dfba187bff69fb7bc4f9a13a0f1d7be3f7049a38c8f0e20ce77ec7d8b"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = diff --git a/docs/reference/api-errors.mdx b/docs/reference/api-errors.mdx index d6c2c07788..03cd9cf206 100644 --- a/docs/reference/api-errors.mdx +++ b/docs/reference/api-errors.mdx @@ -34,6 +34,7 @@ Recognized gateway reasons include the following. | `REQUEST_ID_PAYLOAD_MISMATCH` | `FAILED_PRECONDITION` | Keep the original payload for that request ID. Inspect the original operation before submitting a different one. | | `REQUEST_OUTCOME_UNCERTAIN` | `FAILED_PRECONDITION` | An attempt is admitted but has no confirmed replayable success. Observe resource state and reconcile effects. Do not switch to a new ID to bypass the admission. | | `REQUEST_REPLAY_UNAVAILABLE` | `FAILED_PRECONDITION` | The original scope, resource, interceptor transformation, or fingerprint key is no longer replayable. Reconcile effects; the gateway does not execute the request again. Missing private-key material can also reject admission before work starts. | +| `REQUEST_STREAM_UNAVAILABLE` | `FAILED_PRECONDITION` | The exec attempt terminated, but its output was not stored. Reconcile command effects; the gateway does not launch it again or recreate its stream. | ## Status and retry guidance @@ -184,12 +185,55 @@ The gateway also limits detached admission workers to 64 per process and these request messages to 4 MiB. Supply IDs through generated/raw RPC clients. Curated SDK request-ID helpers and -automatic retry policies are not part of this contract yet. Exec, SSH sessions, +automatic retry policies are not part of this contract yet. SSH sessions, rootfs staging, and supervisor-authenticated mutations do not gain deduplication from this feature. Sandbox principals cannot opt into `UpdateConfig` admission. Check gateway compatibility before relying on IDs; older protobuf servers can silently ignore unknown fields. +## Exec launch admission + +`ExecSandbox` and `ExecSandboxInteractive` accept an optional `request_id` in +`ExecSandboxRequest`. For interactive exec, supply it in the initial `start` +message. The UUID rules, keyed fingerprints, current authorization checks, +per-caller quota, and request-size limit above also apply to exec. + +The gateway admits one owner before opening the supervisor relay. Both RPCs +fingerprint the command, sandbox name, workspace selector, environment, initial +stdin, and other start fields, excluding the request ID. Admission also binds the +resolved sandbox UUID, so a replacement with the same name cannot receive the +original launch. If the sandbox is replaced between authorization lookups for an +unchanged name and workspace selector, the gateway returns +`REQUEST_REPLAY_UNAVAILABLE` before admitting or launching the command. +The RPCs use separate method namespaces. +Later interactive stdin and resize messages are not part of the fingerprint and +cannot be submitted through a duplicate stream. + +| Admitted attempt | Duplicate response | +|---|---| +| Still running, interrupted, timed out, or missing confirmed exit status | `REQUEST_OUTCOME_UNCERTAIN`. The record does not expire or permit another launch. | +| Confirmed remote exit, including a nonzero exit code | `REQUEST_STREAM_UNAVAILABLE` for 24 hours after durable completion. No output, exit code, stream attachment, or stdin is replayed. | +| Changed start payload | `REQUEST_ID_PAYLOAD_MISMATCH`. The changed command does not run. | +| Sandbox replaced under the same name | `REQUEST_REPLAY_UNAVAILABLE`. The replacement does not receive the command. | + +The terminal receipt contains only a marker and completion time, not commands, +environment values, stdin, stdout, or stderr. Receiving a stream handle is not +completion. A gateway-generated timeout exit code of `124` does not prove the +remote process stopped, so it leaves the admission unresolved. A confirmed remote +exit code of `124` is a terminal result. + +Cancellation does not undo admission. Existing transport behavior still applies +to the original execution, including interactive SSH closure on disconnect; the +gateway does not promise that a disconnected command continues. When it loses +exit confirmation, the record remains unresolved even after restart. Current +authorization is checked before duplicate lookup, and the resolved sandbox UUID +must match the original admission before the gateway returns its recorded outcome. + +After confirmed terminal retention expires, the same ID can start a new command. +Do not reuse an old ID without reconciling its effects. Empty IDs keep the existing +streaming behavior without launch deduplication. Use generated/raw RPC clients +for this opt-in contract; curated SDK helpers and automatic retries are separate. + ## Deletion outcomes Delete, membership-removal, and SSH-revocation RPCs return a typed outcome. diff --git a/e2e/python/test_exec_admission.py b/e2e/python/test_exec_admission.py new file mode 100644 index 0000000000..dd5c8a9b72 --- /dev/null +++ b/e2e/python/test_exec_admission.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import threading +import time +import uuid +from typing import TYPE_CHECKING + +import grpc +import pytest +from google.protobuf import duration_pb2 + +from openshell._proto import datamodel_pb2, openshell_pb2 +from openshell.errors import from_grpc_error + +if TYPE_CHECKING: + from collections.abc import Callable + + from openshell import Sandbox, SandboxClient + + +def assert_blocked(stream, reason: str) -> None: + with pytest.raises(grpc.RpcError) as caught: + list(stream) + error = from_grpc_error(caught.value) + assert error.code() == grpc.StatusCode.FAILED_PRECONDITION + assert error.error_info is not None + assert error.error_info.reason == reason + + +@pytest.mark.parametrize("interactive", [False, True]) +def test_exec_request_id_never_relaunches_or_replays_output( + sandbox: Callable[..., Sandbox], sandbox_client: SandboxClient, interactive: bool +) -> None: + with sandbox(delete_on_exit=True) as sb: + path = f"/sandbox/exec-admission-{uuid.uuid4().hex}" + request = openshell_pb2.ExecSandboxRequest( + sandbox=sb.sandbox.name, + workspace_scope=datamodel_pb2.WorkspaceSelector(workspace="default"), + command=[ + "/bin/sh", + "-c", + f"printf x >> {path}; printf started; " + f"while [ ! -e {path}.release ]; do sleep 0.1; done; " + "printf finished; exit 7", + ], + request_id=str(uuid.uuid4()), + ) + done = threading.Event() + + def invoke(value): + if not interactive: + return sandbox_client._stub.ExecSandbox(value, timeout=30) + + def inputs(): + yield openshell_pb2.ExecSandboxInput(start=value) + done.wait(timeout=30) + + return sandbox_client._stub.ExecSandboxInteractive(inputs(), timeout=30) + + stream = invoke(request) + try: + first = next(stream) + assert first.WhichOneof("payload") == "stdout" + assert b"started" in first.stdout.data + assert_blocked(invoke(request), "REQUEST_OUTCOME_UNCERTAIN") + changed = openshell_pb2.ExecSandboxRequest() + changed.CopyFrom(request) + changed.command.append("changed-payload") + assert_blocked(invoke(changed), "REQUEST_ID_PAYLOAD_MISMATCH") + assert sb.exec(["touch", f"{path}.release"]).exit_code == 0 + events = list(stream) + assert events[-1].WhichOneof("payload") == "exit" + assert events[-1].exit.exit_code == 7 + assert_blocked(invoke(request), "REQUEST_STREAM_UNAVAILABLE") + result = sb.exec(["cat", path]) + assert result.exit_code == 0 + assert result.stdout == "x" + finally: + done.set() + stream.cancel() + + +@pytest.mark.parametrize("interactive", [False, True]) +def test_exec_request_timeout_keeps_launch_unresolved( + sandbox: Callable[..., Sandbox], sandbox_client: SandboxClient, interactive: bool +) -> None: + with sandbox(delete_on_exit=True) as sb: + path = f"/sandbox/exec-timeout-{uuid.uuid4().hex}" + request = openshell_pb2.ExecSandboxRequest( + sandbox=sb.sandbox.name, + workspace_scope=datamodel_pb2.WorkspaceSelector(workspace="default"), + command=["/bin/sh", "-c", f"printf x >> {path}; sleep 5"], + execution_timeout=duration_pb2.Duration(seconds=1), + request_id=str(uuid.uuid4()), + ) + done = threading.Event() + + def invoke(): + if not interactive: + return sandbox_client._stub.ExecSandbox(request, timeout=30) + + def inputs(): + yield openshell_pb2.ExecSandboxInput(start=request) + done.wait(timeout=30) + + return sandbox_client._stub.ExecSandboxInteractive(inputs(), timeout=30) + + stream = invoke() + try: + events = [] + for event in stream: + events.append(event) + if event.WhichOneof("payload") == "exit": + # A synthetic timeout leaves the interactive input task + # alive until the client closes its side of the stream. + done.set() + assert events[-1].exit.exit_code == 124 + assert_blocked(invoke(), "REQUEST_OUTCOME_UNCERTAIN") + assert sb.exec(["cat", path]).stdout == "x" + finally: + done.set() + stream.cancel() + + +def test_exec_client_cancellation_does_not_clear_launch_admission( + sandbox: Callable[..., Sandbox], sandbox_client: SandboxClient +) -> None: + with sandbox(delete_on_exit=True) as sb: + path = f"/sandbox/exec-cancel-{uuid.uuid4().hex}" + request = openshell_pb2.ExecSandboxRequest( + sandbox=sb.sandbox.name, + workspace_scope=datamodel_pb2.WorkspaceSelector(workspace="default"), + command=[ + "/bin/sh", + "-c", + f"printf x >> {path}; printf started; sleep 1; printf finished", + ], + request_id=str(uuid.uuid4()), + ) + stream = sandbox_client._stub.ExecSandbox(request, timeout=30) + assert b"started" in next(stream).stdout.data + stream.cancel() + deadline = time.monotonic() + 15 + while True: + with pytest.raises(grpc.RpcError) as caught: + list(sandbox_client._stub.ExecSandbox(request, timeout=30)) + error = from_grpc_error(caught.value) + assert error.error_info is not None + if error.error_info.reason == "REQUEST_STREAM_UNAVAILABLE": + break + assert error.error_info.reason == "REQUEST_OUTCOME_UNCERTAIN" + assert time.monotonic() < deadline + time.sleep(0.1) + assert sb.exec(["cat", path]).stdout == "x" diff --git a/proto/openshell.proto b/proto/openshell.proto index dd05f0fd2b..ed18c86b25 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -1884,6 +1884,12 @@ message ExecSandboxRequest { // sourced by them) are applied. When true, the command runs without those // files (`bash -c`), for automation that needs predictable startup behavior. bool no_login_shell = 10; + + // Optional nonzero UUID for durable launch admission. Also applies to the + // initial ExecSandboxInteractive start message. Duplicates never relaunch, + // reattach, or replay output/stdin. Unconfirmed executions remain fenced; + // confirmed terminal executions retain the fence for 24 hours. + string request_id = 11; } // One stdout chunk from a sandbox exec. diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 89e10bd26c..e40dbd8082 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -6655,7 +6655,12 @@ type ExecSandboxRequest struct { // (`bash -lc`) so user startup files (.bash_profile/.profile, and .bashrc if // sourced by them) are applied. When true, the command runs without those // files (`bash -c`), for automation that needs predictable startup behavior. - NoLoginShell bool `protobuf:"varint,10,opt,name=no_login_shell,json=noLoginShell,proto3" json:"no_login_shell,omitempty"` + NoLoginShell bool `protobuf:"varint,10,opt,name=no_login_shell,json=noLoginShell,proto3" json:"no_login_shell,omitempty"` + // Optional nonzero UUID for durable launch admission. Also applies to the + // initial ExecSandboxInteractive start message. Duplicates never relaunch, + // reattach, or replay output/stdin. Unconfirmed executions remain fenced; + // confirmed terminal executions retain the fence for 24 hours. + RequestId string `protobuf:"bytes,11,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6767,6 +6772,13 @@ func (x *ExecSandboxRequest) GetNoLoginShell() bool { return false } +func (x *ExecSandboxRequest) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + // One stdout chunk from a sandbox exec. type ExecSandboxStdout struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -17971,7 +17983,7 @@ const file_openshell_proto_rawDesc = "" + "\x05token\x18\x01 \x01(\tB\x04\x88\xb5\x18\x01R\x05token\x12#\n" + "\rallow_missing\x18\x02 \x01(\bR\fallowMissing\"b\n" + "\x18RevokeSshSessionResponse\x127\n" + - "\aoutcome\x18\x02 \x01(\x0e2\x1d.openshell.v1.DeletionOutcomeR\aoutcomeJ\x04\b\x01\x10\x02R\arevoked\"\xa0\x04\n" + + "\aoutcome\x18\x02 \x01(\x0e2\x1d.openshell.v1.DeletionOutcomeR\aoutcomeJ\x04\b\x01\x10\x02R\arevoked\"\xbf\x04\n" + "\x12ExecSandboxRequest\x12R\n" + "\x0fworkspace_scope\x18\f \x01(\v2).openshell.datamodel.v1.WorkspaceSelectorR\x0eworkspaceScope\x12\x18\n" + "\asandbox\x18\x01 \x01(\tR\asandbox\x12\x18\n" + @@ -17984,7 +17996,9 @@ const file_openshell_proto_rawDesc = "" + "\x04cols\x18\b \x01(\rR\x04cols\x12\x12\n" + "\x04rows\x18\t \x01(\rR\x04rows\x12$\n" + "\x0eno_login_shell\x18\n" + - " \x01(\bR\fnoLoginShell\x1a>\n" + + " \x01(\bR\fnoLoginShell\x12\x1d\n" + + "\n" + + "request_id\x18\v \x01(\tR\trequestId\x1a>\n" + "\x10EnvironmentEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01J\x04\b\x05\x10\x06R\x0ftimeout_seconds\"'\n" + From 49df4d7ca7bd7a2e53d2593da1f904bcd54598a6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 22 Sep 2026 22:29:54 +0000 Subject: [PATCH 6/9] fix(server): drain supervisor ownership cleanup on shutdown (#3547) Closes #3546 Signed-off-by: Evan Lezar --- architecture/gateway.md | 9 + crates/openshell-server/src/lib.rs | 17 +- .../src/supervisor_session.rs | 228 +++++++++++++++++- docs/reference/sandbox-compute-drivers.mdx | 10 +- skills/debug-openshell-cluster/SKILL.md | 6 + 5 files changed, 258 insertions(+), 12 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index 413c126552..e065fd8e1a 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -1044,6 +1044,15 @@ The same relay pattern backs interactive SSH, command execution, file sync, and local service forwarding. The gateway tracks live sessions in memory and persists session records so tokens can expire or be revoked. +Graceful gateway shutdown closes supervisor-session admission before stopping +local compute. It then signals the remaining control sessions to exit and waits +up to ten seconds for their cleanup, including conditional deletion of persisted +ownership. Pending connection setup and sessions already removed from the live +registry remain tracked until cleanup finishes. This lets a replacement +supervisor claim ownership immediately after restart without deleting a newer +replica's claim. An incomplete drain is reported as a shutdown error. Closing +these control sessions does not stop Kubernetes-owned workloads. + Relay liveness has two backstops so a reset supervisor session cannot leave a request parked forever. The gateway runs server-side HTTP/2 keepalive on supervisor connections, and each exec relay's SSH client uses SSH keepalive: an diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 4a8f2d5b35..5a43364247 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1016,17 +1016,26 @@ pub(crate) async fn run_server( shutdown_signal().await; info!("Shutdown signal received; stopping gateway"); state.gateway_shutting_down.store(true, Ordering::Release); + state.supervisor_sessions.close_admission(); let _ = shutdown_tx.send(true); if let Err(err) = listener_task.await { warn!(error = %err, "Gateway listener task failed during shutdown"); } - state - .compute - .cleanup_on_shutdown() - .await + let compute_cleanup = state.compute.cleanup_on_shutdown().await; + // A stopped supervisor may still have a detached task deleting its owner + // record. Drain it even when compute cleanup failed before exiting Tokio. + let session_cleanup = state + .supervisor_sessions + .shutdown(Duration::from_secs(10)) + .await; + if let Err(err) = &session_cleanup { + warn!(error = %err, "Gateway supervisor session cleanup incomplete"); + } + compute_cleanup .map_err(|err| Error::execution(format!("gateway shutdown cleanup failed: {err}")))?; + session_cleanup.map_err(Error::execution)?; Ok(()) } diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index a1140e9fd3..710ecc1406 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -3,11 +3,11 @@ use std::collections::HashMap; use std::pin::Pin; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{OwnedRwLockReadGuard, RwLock, mpsc, oneshot, watch}; use tokio_stream::wrappers::ReceiverStream; use tonic::metadata::{Ascii, MetadataValue}; use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity}; @@ -20,8 +20,8 @@ use openshell_core::proto::{ PeerRelayFrame, PeerRelayInit, ProviderReadinessObservation, RelayFrame, RelayInit, RelayOpen, ReportEndpointStatusRequest, ReportEndpointStatusResponse, ReportMainProcessExitRequest, ReportMainProcessExitResponse, ReportProviderReadinessRequest, ReportProviderReadinessResponse, - Sandbox, SandboxPhase, SessionAccepted, SshRelayTarget, SupervisorMessage, gateway_message, - open_shell_client, peer_relay_frame, relay_open, supervisor_message, + Sandbox, SandboxPhase, SessionAccepted, SshRelayTarget, SupervisorHello, SupervisorMessage, + gateway_message, open_shell_client, peer_relay_frame, relay_open, supervisor_message, }; use openshell_core::transport_errors::is_expected_transport_close_status; @@ -324,6 +324,11 @@ pub struct SupervisorSessionRegistry { sessions: Mutex>, /// `channel_id` -> oneshot sender for the reverse CONNECT stream. pending_relays: Mutex>, + /// Read guards cover owner publication through final cleanup, even after + /// a session has left `sessions`. Shutdown waits for the write guard. + session_lifetimes: Arc>, + admission_closed: AtomicBool, + shutdown: watch::Sender, } struct PendingRelay { @@ -355,6 +360,36 @@ impl SupervisorSessionRegistry { Self::default() } + fn track_session(&self) -> Result, Status> { + // Acquire tracking BEFORE checking admission. A concurrent shutdown + // either sees this reader and waits for it, or prevents its admission. + let lifetime = Arc::clone(&self.session_lifetimes) + .try_read_owned() + .map_err(|_| Status::unavailable("gateway is shutting down"))?; + if self.admission_closed.load(Ordering::Acquire) { + return Err(Status::unavailable("gateway is shutting down")); + } + Ok(lifetime) + } + + /// Prevent existing HTTP connections from creating new owner records. + pub(crate) fn close_admission(&self) { + self.admission_closed.store(true, Ordering::Release); + } + + /// Close control sessions and wait for tracked ownership cleanup. Call + /// after compute shutdown so supervisors can finish normal stop reporting. + pub(crate) async fn shutdown(&self, timeout: Duration) -> Result<(), String> { + self.close_admission(); + self.shutdown.send_replace(true); + tokio::time::timeout(timeout, self.session_lifetimes.write()) + .await + .map(|_| ()) + .map_err(|_| { + format!("supervisor session ownership cleanup did not complete within {timeout:?}") + }) + } + /// Register a live supervisor session for the given sandbox. /// /// If a previous session exists for the same sandbox, its shutdown signal @@ -1761,6 +1796,34 @@ pub async fn handle_connect_supervisor( // supervisors remain usable but cannot assert provider installation. let provider_readiness = ProviderReadinessEvidence::from_hello(&hello)?; + let session_lifetime = state.supervisor_sessions.track_session()?; + let state = Arc::clone(state); + // Keep setup alive if the RPC caller disconnects after publication. The + // tracking guard moves into the session task or outlives early cleanup. + tokio::spawn(establish_supervisor_session( + state, + inbound, + hello, + provider_readiness, + session_lifetime, + )) + .await + .map_err(|error| Status::internal(format!("supervisor session setup failed: {error}")))? +} + +async fn establish_supervisor_session( + state: Arc, + mut inbound: tonic::Streaming, + hello: SupervisorHello, + provider_readiness: ProviderReadinessEvidence, + session_lifetime: OwnedRwLockReadGuard<()>, +) -> Result< + Response< + Pin> + Send + 'static>>, + >, + Status, +> { + let sandbox_id = hello.sandbox_id.clone(); let session_id = Uuid::new_v4().to_string(); let owner_peer_endpoint = state.peer_endpoint.as_deref().map_or_else( || local_owner_endpoint(&state.replica_id), @@ -1808,7 +1871,7 @@ pub async fn handle_connect_supervisor( // results before acknowledging the session so it cannot inherit evidence // reported by the superseded stream. if let Err(error) = crate::grpc::policy::reset_endpoint_status_for_supervisor_session( - state, + &state, &sandbox_id, &session_id, ) @@ -1905,9 +1968,10 @@ pub async fn handle_connect_supervisor( } // Step 4: Spawn the session loop that reads inbound messages. - let state_clone = Arc::clone(state); + let state_clone = Arc::clone(&state); let sandbox_id_clone = sandbox_id.clone(); tokio::spawn(async move { + let _session_lifetime = session_lifetime; let mut owner_guard = owner_guard; run_session_loop( &state_clone, @@ -2030,6 +2094,7 @@ async fn run_session_loop( mut shutdown_rx: oneshot::Receiver<()>, owner_guard: &mut OwnerGuard, ) { + let mut gateway_shutdown = state.supervisor_sessions.shutdown.subscribe(); let heartbeat_interval = Duration::from_secs(u64::from(HEARTBEAT_INTERVAL_SECS)); let mut heartbeat_timer = tokio::time::interval(heartbeat_interval); // Skip the first immediate tick. @@ -2037,6 +2102,10 @@ async fn run_session_loop( loop { tokio::select! { + () = async { let _ = gateway_shutdown.wait_for(|shutdown| *shutdown).await; } => { + info!(sandbox_id = %sandbox_id, session_id = %session_id, "supervisor session: gateway shutting down"); + break; + } _ = &mut shutdown_rx => { info!(sandbox_id = %sandbox_id, session_id = %session_id, "supervisor session: superseded by reconnect, shutting down"); break; @@ -2214,6 +2283,153 @@ mod tests { oneshot::channel::<()>().0 } + #[tokio::test] + async fn shutdown_waits_for_owner_release_after_session_leaves_registry() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let owner_index = Arc::new(SupervisorOwnerIndex::new(test_store().await, OWNER_TTL)); + let lifetime = registry.track_session().unwrap(); + let owner = owner_index + .publish( + "sb-1", + "old-session", + "old-instance", + 1, + "old-replica", + "local://old", + ) + .await + .unwrap(); + let (tx, _rx) = mpsc::channel(1); + registry.register("sb-1".into(), "old-session".into(), tx, make_shutdown()); + + let (removed_tx, removed_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let cleanup_registry = Arc::clone(®istry); + let cleanup_index = owner_index.clone(); + let cleanup = tokio::spawn(async move { + let _lifetime = lifetime; + cleanup_registry.remove_if_current("sb-1", "old-session"); + removed_tx.send(()).unwrap(); + release_rx.await.unwrap(); + cleanup_index.release_if_current(&owner).await.unwrap(); + }); + removed_rx.await.unwrap(); + assert!(registry.sessions.lock().unwrap().is_empty()); + + let shutdown = registry.shutdown(Duration::from_secs(5)); + tokio::pin!(shutdown); + assert!(futures_util::poll!(&mut shutdown).is_pending()); + assert!(matches!( + owner_index + .publish( + "sb-1", + "new-session", + "new-instance", + 1, + "new-replica", + "local://new" + ) + .await, + Err(OwnerError::AlreadyOwned) + )); + + release_tx.send(()).unwrap(); + shutdown.await.unwrap(); + cleanup.await.unwrap(); + assert!(owner_index.read("sb-1").await.unwrap().is_none()); + owner_index + .publish( + "sb-1", + "new-session", + "new-instance", + 1, + "new-replica", + "local://new", + ) + .await + .expect("restart can immediately claim ownership after shutdown"); + } + + #[tokio::test] + async fn shutdown_waits_for_admitted_setup_and_rejects_new_connections() { + let registry = SupervisorSessionRegistry::new(); + // An RPC has been admitted but has not yet registered a live session. + let lifetime = registry.track_session().unwrap(); + registry.close_admission(); + assert_eq!( + registry.track_session().unwrap_err().code(), + tonic::Code::Unavailable + ); + // Closing admission alone must leave stop reporting available. + assert!(!*registry.shutdown.borrow()); + + let shutdown = registry.shutdown(Duration::from_secs(5)); + tokio::pin!(shutdown); + assert!(futures_util::poll!(&mut shutdown).is_pending()); + // An admitted setup that starts its session loop after cancellation + // must still observe the shutdown signal immediately. + let mut late_subscriber = registry.shutdown.subscribe(); + assert!(*late_subscriber.wait_for(|closing| *closing).await.unwrap()); + drop(lifetime); + shutdown.await.unwrap(); + assert_eq!( + registry.track_session().unwrap_err().code(), + tonic::Code::Unavailable + ); + } + + #[tokio::test] + async fn shutdown_cleanup_does_not_delete_replacement_owner() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let owner_index = SupervisorOwnerIndex::new(test_store().await, OWNER_TTL); + let lifetime = registry.track_session().unwrap(); + let old = owner_index + .publish("sb-1", "old", "instance", 1, "replica-a", "local://a") + .await + .unwrap(); + let replacement = owner_index + .publish("sb-1", "new", "instance", 2, "replica-b", "local://b") + .await + .unwrap(); + let shutdown = registry.shutdown(Duration::from_secs(5)); + tokio::pin!(shutdown); + assert!(futures_util::poll!(&mut shutdown).is_pending()); + owner_index.release_if_current(&old).await.unwrap(); + drop(lifetime); + shutdown.await.unwrap(); + let persisted = owner_index.read("sb-1").await.unwrap().unwrap(); + assert_eq!(persisted.session_id, replacement.session_id); + assert_eq!(persisted.owner_replica_id, replacement.owner_replica_id); + } + + #[tokio::test] + async fn shutdown_reports_bounded_failure_when_cleanup_stalls() { + let registry = SupervisorSessionRegistry::new(); + let lifetime = registry.track_session().unwrap(); + let error = registry + .shutdown(Duration::from_millis(10)) + .await + .unwrap_err(); + assert!(error.contains("ownership cleanup did not complete")); + assert!(*registry.shutdown.borrow()); + assert_eq!( + registry.track_session().unwrap_err().code(), + tonic::Code::Unavailable + ); + drop(lifetime); + registry.shutdown(Duration::from_secs(1)).await.unwrap(); + } + + #[tokio::test] + async fn shutdown_without_sessions_completes_and_closes_admission() { + let registry = SupervisorSessionRegistry::new(); + registry.shutdown(Duration::from_secs(1)).await.unwrap(); + assert_eq!( + registry.track_session().unwrap_err().code(), + tonic::Code::Unavailable + ); + } + #[test] fn peer_tls_client_config_requires_certificate_and_key_together() { let config = PeerTlsClientConfig { diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 8919dc554d..5bca34af16 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -33,13 +33,19 @@ Delete remains independent and removes compute plus driver-owned persistent state. While a sandbox is stopped, gateway access paths and exposed services remain unavailable. -Restarting the gateway preserves this intent. The gateway does not stop Docker -or Podman containers during shutdown. At startup it sends idempotent start +Restarting the gateway preserves this intent. During graceful shutdown, the +gateway stops running-intent Docker, Podman, and MicroVM sandboxes through their +drivers without recording an explicit user stop. At startup it sends idempotent start requests for Docker, Podman, and MicroVM sandboxes that were intended to run; already-running resources are unchanged, retained stopped compute is restarted, and explicitly stopped sandboxes remain stopped. Kubernetes workloads continue running independently of the gateway process. +Before exiting, the gateway waits up to ten seconds for supervisor-session +ownership cleanup so replacement supervisors can reconnect after restart. +If tracked cleanup exceeds this deadline, the gateway reports a shutdown +error. Persistence errors during ownership release are logged separately. + The gateway forwards one exact, persisted main-process specification to every driver. Drivers serialize that specification in `OPENSHELL_MAIN_PROCESS_SPEC`; they do not install an idle `sleep` workload or diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 420ae24e8d..e331c605e9 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -289,6 +289,12 @@ not follow this local gateway lifecycle. Internal and external drivers follow the same rule: `GetCapabilities.gateway_manages_lifecycle` must be true for the gateway to run shutdown and startup sweeps. +The gateway also drains supervisor-session ownership cleanup before exiting. +If shutdown reports `Gateway supervisor session cleanup incomplete`, inspect +the associated persistence errors: a stopped supervisor's owner record may +remain until its lease expires and temporarily block reconnection. Successful +compute stop alone does not confirm that session cleanup finished. + ### Step 5: Check Podman-Backed Gateways ```bash From feff8979682b62c9bac2f32609abe9ed53f39ad0 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 23:04:15 +0000 Subject: [PATCH 7/9] fix(sandbox): await SFTP writes before acknowledging (#3568) Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/sftp.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/openshell-sandbox/src/sftp.rs b/crates/openshell-sandbox/src/sftp.rs index 2cf02a8390..2c0e408de4 100644 --- a/crates/openshell-sandbox/src/sftp.rs +++ b/crates/openshell-sandbox/src/sftp.rs @@ -320,6 +320,11 @@ impl russh_sftp::server::Handler for SftpHandler { .await .map_err(io_status)?; file.write_all(&data).await.map_err(io_status)?; + // `tokio::fs::File::write_all` may return after copying the bytes into + // its internal buffer while the blocking filesystem write is still in + // flight. Do not acknowledge the SFTP write until that operation has + // completed, otherwise session shutdown can race the final write. + file.flush().await.map_err(io_status)?; Ok(ok(id)) } From 84960e70a3ab4ad08596095ae4da1b90ac895449 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 23 Sep 2026 00:02:10 +0000 Subject: [PATCH 8/9] fix(kubernetes): scope resource admission RBAC (#3571) * fix(kubernetes): scope resource admission RBAC Signed-off-by: Drew Newberry * fix(helm): gate PVC admission reads Signed-off-by: Drew Newberry --------- Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 7 +- architecture/compute-runtimes.md | 6 +- crates/openshell-driver-kubernetes/README.md | 4 +- .../openshell-driver-kubernetes/src/driver.rs | 74 +------------------ .../src/resource_admission.rs | 72 +++++++----------- deploy/helm/openshell-workspace/README.md | 6 +- .../helm/openshell-workspace/README.md.gotmpl | 5 +- .../openshell-workspace/templates/role.yaml | 6 +- .../tests/workspace_test.yaml | 30 ++++++++ deploy/helm/openshell-workspace/values.yaml | 3 + .../helm/openshell/templates/clusterrole.yaml | 4 +- deploy/helm/openshell/templates/role.yaml | 6 +- .../openshell/tests/clusterrole_test.yaml | 35 +++++++++ .../tests/sandbox_namespace_test.yaml | 30 ++++++++ docs/kubernetes/setup.mdx | 5 ++ docs/reference/gateway-config.mdx | 48 ++++++------ skills/debug-openshell-cluster/SKILL.md | 15 ++-- 17 files changed, 191 insertions(+), 165 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 2df971436f..cb3023f476 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -72,9 +72,10 @@ mise run helm:skaffold:run Resource admission defaults to enabled and caller driver config to disabled. Driver-config scenarios need an explicit `allowDriverConfig` opt-in; external attachments also need administrator-controlled approval labels in the target -namespace. GPU attachments are exempt from labels. Managed workspace image-pull -Secrets are copied from an approved source in the gateway namespace; do not grant -approval to the gateway database PVC or disable admission to make tests pass. +namespace. GPU attachments and operator-selected image-pull Secrets are exempt +from labels. Managed workspace image-pull Secrets are copied from the configured +source in the gateway namespace; do not grant approval to the gateway database +PVC or disable admission to make tests pass. The Skaffold flow builds distinct `gateway`, `sandbox`, and `supervisor` images and deploys the OpenShell Helm chart. The Kubernetes driver creates a diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 44415009c8..3f28c3f0a4 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -465,9 +465,9 @@ management. RBAC uses a namespace-scoped Role. Each new namespace receives a ServiceAccount and the configured gateway-only SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the driver's source namespace on every sandbox create so registry credential -rotations propagate. Resource admission first validates the source as shared -operator infrastructure, and copies carry gateway and workspace ownership -labels; an unrelated existing target is never adopted. The namespace also copies +rotations propagate. Their names are operator-selected gateway configuration, +not caller attachments. Copies carry gateway and workspace ownership labels; an +unrelated existing target is never adopted. The namespace also copies OpenShift SCC UID-range and supplemental-group annotations from the gateway namespace when present. The driver deletes the namespace during workspace deletion. The workspace remains durably `Terminating` until the Kubernetes API diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 6e1578cee4..d21a6c1ff1 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -5,8 +5,8 @@ Kubernetes-backed compute driver for OpenShell cluster deployments. Caller driver config is disabled by default. External resource references need administrator-controlled approval labels in every workspace mode, including before restart and scheduling-gate release. GPU devices are temporarily exempt. -Managed workspace image-pull Secrets are copied only after the configured source -Secret passes shared-resource admission; copies carry gateway ownership metadata. +Image-pull Secrets are operator-selected gateway configuration rather than caller +attachments. Managed workspace copies carry gateway ownership metadata. See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). The driver uses the Kubernetes API to create, delete, fetch, and watch sandbox diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 356fdc249d..5d072a0f21 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1306,28 +1306,6 @@ impl KubernetesComputeDriver { } }; - self.config - .resource_admission - .admit_shared( - source - .metadata - .labels - .as_ref() - .into_iter() - .flat_map(|labels| labels.iter()), - ) - .map_err(|error| { - admission_error(tonic::Status::new( - error.code(), - format!( - "Secret '{}/{}': {}", - self.config.namespace, - secret_name, - error.message() - ), - )) - })?; - let existing = tokio::time::timeout(KUBE_API_TIMEOUT, target_api.get_opt(secret_name)) .await .map_err(|_| { @@ -11070,7 +11048,7 @@ mod tests { } #[tokio::test] - async fn managed_image_pull_secret_admits_source_before_copying() { + async fn managed_image_pull_secret_copies_operator_selected_source() { let source_path = "/api/v1/namespaces/openshell/secrets/regcred"; let target_path = "/api/v1/namespaces/managed-team-a/secrets/regcred"; let copied = serde_json::json!({ @@ -11101,7 +11079,7 @@ mod tests { "metadata": { "name": "regcred", "namespace": "openshell", - "labels": {"openshell.ai/sandbox-attachable": "true"} + "labels": {} }, "type": "kubernetes.io/dockerconfigjson", "data": { ".dockerconfigjson": "e30=" } @@ -11150,56 +11128,10 @@ mod tests { driver .ensure_image_pull_secrets("managed-team-a", "team-a") .await - .expect("approved source should be copied"); + .expect("operator-selected source should be copied"); assert!(steps.lock().unwrap().is_empty()); } - #[tokio::test] - async fn managed_image_pull_secret_rejects_unapproved_source_without_copying() { - let service = tower::service_fn( - move |request: http::Request| async move { - assert_eq!(request.method(), http::Method::GET); - assert_eq!( - request.uri().path(), - "/api/v1/namespaces/openshell/secrets/regcred" - ); - Ok::<_, std::convert::Infallible>(kube_test_response( - http::StatusCode::OK, - serde_json::json!({ - "apiVersion": "v1", - "kind": "Secret", - "metadata": {"name": "regcred", "namespace": "openshell"}, - "type": "kubernetes.io/dockerconfigjson", - "data": { ".dockerconfigjson": "e30=" } - }), - )) - }, - ); - let client = Client::new(service, "openshell"); - let driver = KubernetesComputeDriver { - client: client.clone(), - watch_client: client, - sandbox_api_version: Arc::new(OnceCell::new()), - config: KubernetesComputeConfig { - namespace: "openshell".into(), - gateway_id: "gateway-a".into(), - image_pull_secrets: vec!["regcred".into()], - ..Default::default() - }, - operator_allowlist: None, - }; - - let error = driver - .ensure_image_pull_secrets("managed-team-a", "team-a") - .await - .expect_err("unapproved source must not be copied"); - assert!(matches!(error, KubernetesDriverError::Precondition(_))); - assert!( - error.to_string().contains("Secret 'openshell/regcred'"), - "unexpected error: {error}" - ); - } - #[test] fn namespace_owned_with_correct_labels() { let labels = BTreeMap::from([ diff --git a/crates/openshell-driver-kubernetes/src/resource_admission.rs b/crates/openshell-driver-kubernetes/src/resource_admission.rs index fa288c4f80..339572d1f2 100644 --- a/crates/openshell-driver-kubernetes/src/resource_admission.rs +++ b/crates/openshell-driver-kubernetes/src/resource_admission.rs @@ -110,9 +110,9 @@ fn inventory(spec: &Value, private_secret: &str) -> Result, { return Err(deny()); } - for secret in spec["imagePullSecrets"].as_array().into_iter().flatten() { - reference(&mut refs, "Secret", secret["name"].as_str(), Scope::Shared); - } + // Image-pull Secrets are selected by gateway configuration rather than by + // the sandbox caller. The kubelet resolves them; they are not workload data + // attachments and do not participate in caller resource admission. for volume in spec["volumes"].as_array().into_iter().flatten() { let object = volume.as_object().ok_or_else(deny)?; let sources: Vec<_> = object.keys().filter(|key| key.as_str() != "name").collect(); @@ -127,50 +127,27 @@ fn inventory(spec: &Value, private_secret: &str) -> Result, volume["persistentVolumeClaim"]["claimName"].as_str(), Scope::Workspace, ), - "secret" => { - let name = volume["secret"]["secretName"].as_str(); - if name != Some(private_secret) { - reference(&mut refs, "Secret", name, Scope::Workspace); - } - } - "configMap" => reference( - &mut refs, - "ConfigMap", - volume["configMap"]["name"].as_str(), - Scope::Workspace, - ), + "secret" if volume["secret"]["secretName"].as_str() == Some(private_secret) => {} + // The typed Kubernetes driver config does not expose Secret or + // ConfigMap volumes. The closed-set fallback rejects them instead + // of expanding gateway RBAC for attachments callers cannot request. _ => return Err(deny()), } } for field in ["containers", "initContainers", "ephemeralContainers"] { for container in spec[field].as_array().into_iter().flatten() { - for env in container["envFrom"].as_array().into_iter().flatten() { - reference( - &mut refs, - "Secret", - env["secretRef"]["name"].as_str(), - Scope::Workspace, - ); - reference( - &mut refs, - "ConfigMap", - env["configMapRef"]["name"].as_str(), - Scope::Workspace, - ); + if container["envFrom"] + .as_array() + .is_some_and(|env| !env.is_empty()) + { + return Err(deny()); } for env in container["env"].as_array().into_iter().flatten() { - reference( - &mut refs, - "Secret", - env["valueFrom"]["secretKeyRef"]["name"].as_str(), - Scope::Workspace, - ); - reference( - &mut refs, - "ConfigMap", - env["valueFrom"]["configMapKeyRef"]["name"].as_str(), - Scope::Workspace, - ); + if !env["valueFrom"]["secretKeyRef"].is_null() + || !env["valueFrom"]["configMapKeyRef"].is_null() + { + return Err(deny()); + } } for field in ["requests", "limits"] { for (resource, _) in container["resources"][field] @@ -207,8 +184,6 @@ pub async fn admit( for reference in inventory(spec, private_secret)? { let (group, version, plural, cluster) = match reference.kind { "PersistentVolumeClaim" => ("", "v1", "persistentvolumeclaims", false), - "Secret" => ("", "v1", "secrets", false), - "ConfigMap" => ("", "v1", "configmaps", false), "RuntimeClass" => ("node.k8s.io", "v1", "runtimeclasses", true), "PriorityClass" => ("scheduling.k8s.io", "v1", "priorityclasses", true), _ => unreachable!("closed resource inventory"), @@ -479,13 +454,12 @@ mod tests { } #[test] - fn inventories_all_containers_and_reference_aliases() { + fn inventories_supported_external_resources() { let pod = serde_json::json!({"automountServiceAccountToken":false,"runtimeClassName":"r","priorityClassName":"p", "volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"gateway-db","readOnly":true}}], - "initContainers":[{"envFrom":[{"secretRef":{"name":"secret"}}]}], - "containers":[{"env":[{"valueFrom":{"configMapKeyRef":{"name":"config"}}}]}]}); + "imagePullSecrets":[{"name":"regcred"}]}); let refs = inventory(&pod, "private").unwrap(); - assert_eq!(refs.len(), 5); + assert_eq!(refs.len(), 3); assert!( refs.iter() .any(|r| r.name == "gateway-db" && r.scope == Scope::Workspace) @@ -497,9 +471,13 @@ mod tests { } #[test] fn rejects_unsupported_volume_sources_but_allows_gpu() { - for kind in ["hostPath", "csi", "projected", "image"] { + for kind in ["hostPath", "csi", "projected", "image", "configMap"] { assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"volumes":[{"name":"x",kind:{}}]}), "private").is_err()); } + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"volumes":[{"name":"x","secret":{"secretName":"external"}}]}), "private").is_err()); + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"volumes":[{"name":"x","secret":{"secretName":"private"}}]}), "private").is_ok()); + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"containers":[{"envFrom":[{"secretRef":{"name":"external"}}]}]}), "private").is_err()); + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"containers":[{"env":[{"valueFrom":{"configMapKeyRef":{"name":"external"}}}]}]}), "private").is_err()); assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"containers":[{"resources":{"limits":{"nvidia.com/gpu":"1"}}}]}), "private").is_ok()); } #[test] diff --git a/deploy/helm/openshell-workspace/README.md b/deploy/helm/openshell-workspace/README.md index 96ae4ca6e4..e3a21b3b34 100644 --- a/deploy/helm/openshell-workspace/README.md +++ b/deploy/helm/openshell-workspace/README.md @@ -27,13 +27,17 @@ helm install openshell-workspace ./deploy/helm/openshell-workspace \ Keep `sandboxServiceAccount.name` aligned with the gateway chart's `sandboxServiceAccount.name`. The defaults for both charts are -`openshell-sandbox`. +`openshell-sandbox`. If the gateway enables +`server.drivers.kubernetes.allowDriverConfig`, also set +`gateway.allowDriverConfig=true` in every workspace release so caller-selected +PVCs can be admitted. ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| | fullnameOverride | string | `""` | Override the full generated resource name. | +| gateway.allowDriverConfig | bool | `false` | Grant PVC metadata reads required when the gateway enables caller driver config. Keep this aligned with server.drivers.kubernetes.allowDriverConfig in the gateway chart. | | gateway.networkPolicy.podSelector | object | `{"app.kubernetes.io/instance":"openshell","app.kubernetes.io/name":"openshell"}` | Labels selecting gateway pods allowed to reach sandbox SSH. | | gateway.serviceAccount.name | string | `"openshell"` | Name of the shared gateway ServiceAccount. | | gateway.serviceAccount.namespace | string | `"openshell"` | Namespace containing the shared gateway ServiceAccount. | diff --git a/deploy/helm/openshell-workspace/README.md.gotmpl b/deploy/helm/openshell-workspace/README.md.gotmpl index a233eef165..a691aba4a2 100644 --- a/deploy/helm/openshell-workspace/README.md.gotmpl +++ b/deploy/helm/openshell-workspace/README.md.gotmpl @@ -27,7 +27,10 @@ helm install openshell-workspace ./deploy/helm/openshell-workspace \ Keep `sandboxServiceAccount.name` aligned with the gateway chart's `sandboxServiceAccount.name`. The defaults for both charts are -`openshell-sandbox`. +`openshell-sandbox`. If the gateway enables +`server.drivers.kubernetes.allowDriverConfig`, also set +`gateway.allowDriverConfig=true` in every workspace release so caller-selected +PVCs can be admitted. {{ template "chart.valuesSection" . }} {{ template "helm-docs.versionFooter" . }} diff --git a/deploy/helm/openshell-workspace/templates/role.yaml b/deploy/helm/openshell-workspace/templates/role.yaml index 3df8928496..bd0e33bad6 100644 --- a/deploy/helm/openshell-workspace/templates/role.yaml +++ b/deploy/helm/openshell-workspace/templates/role.yaml @@ -10,10 +10,12 @@ metadata: labels: {{- include "openshell-workspace.labels" . | nindent 4 }} rules: - # Metadata-only application admission still requires Kubernetes get permission. + {{- if .Values.gateway.allowDriverConfig }} + # Metadata-only admission of caller-selected PVCs requires get permission. - apiGroups: [""] - resources: ["persistentvolumeclaims", "configmaps", "secrets"] + resources: ["persistentvolumeclaims"] verbs: ["get"] + {{- end }} - apiGroups: - agents.x-k8s.io resources: diff --git a/deploy/helm/openshell-workspace/tests/workspace_test.yaml b/deploy/helm/openshell-workspace/tests/workspace_test.yaml index 2f71920eaf..7358e7de38 100644 --- a/deploy/helm/openshell-workspace/tests/workspace_test.yaml +++ b/deploy/helm/openshell-workspace/tests/workspace_test.yaml @@ -20,6 +20,36 @@ tests: - equal: path: metadata.namespace value: app-a + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + + - it: grants PVC admission reads only when caller driver config is enabled + template: templates/role.yaml + set: + gateway.allowDriverConfig: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] - it: binds the shared gateway service account template: templates/rolebinding.yaml diff --git a/deploy/helm/openshell-workspace/values.yaml b/deploy/helm/openshell-workspace/values.yaml index 2c52b0460d..069e573e2f 100644 --- a/deploy/helm/openshell-workspace/values.yaml +++ b/deploy/helm/openshell-workspace/values.yaml @@ -7,6 +7,9 @@ nameOverride: "" fullnameOverride: "" gateway: + # -- Grant PVC metadata reads required when the gateway enables caller driver config. + # Keep this aligned with server.drivers.kubernetes.allowDriverConfig in the gateway chart. + allowDriverConfig: false serviceAccount: # -- Name of the shared gateway ServiceAccount. name: openshell diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 1df5c52467..8bd68eeb5c 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -15,9 +15,9 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get"] - {{- if ne $workspaceMode "shared" }} + {{- if and (ne $workspaceMode "shared") .Values.server.drivers.kubernetes.allowDriverConfig }} - apiGroups: [""] - resources: ["persistentvolumeclaims", "configmaps", "secrets"] + resources: ["persistentvolumeclaims"] verbs: ["get"] {{- end }} # Validate projected ServiceAccount tokens during sandbox bootstrap and diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 365ef3e9c7..d43f5de0ab 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -10,10 +10,12 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: - # Metadata-only application admission still requires Kubernetes get permission. + {{- if .Values.server.drivers.kubernetes.allowDriverConfig }} + # Metadata-only admission of caller-selected PVCs requires get permission. - apiGroups: [""] - resources: ["persistentvolumeclaims", "configmaps", "secrets"] + resources: ["persistentvolumeclaims"] verbs: ["get"] + {{- end }} - apiGroups: - agents.x-k8s.io resources: diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 9639232c32..528efd7cba 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -26,6 +26,41 @@ tests: resources: ["networkpolicies"] verbs: ["get", "create", "patch", "update"] + - it: limits namespaced admission reads to PVCs + set: + server.drivers.kubernetes.workspaceMode: managed + server.drivers.kubernetes.allowDriverConfig: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + + - it: omits PVC admission reads when caller driver config is disabled + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - it: preserves sandbox-runtime fence permissions when gateway isolation is disabled set: server.drivers.kubernetes.workspaceMode: managed diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 6ec7ae4d65..36c2276e91 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -74,6 +74,36 @@ tests: apiGroups: [""] resources: ["secrets"] verbs: ["create", "list", "delete"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + + - it: grants PVC admission reads only when caller driver config is enabled + template: templates/role.yaml + set: + server.drivers.kubernetes.allowDriverConfig: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] - it: uses explicit sandboxNamespace for sandbox RoleBinding template: templates/rolebinding.yaml diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 43e221b824..494e14aa22 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -132,6 +132,11 @@ provisioned namespaces, use `workspaceMode=operator`, configure exactly one of `operatorNamespaceLabel` or `operatorNamespaceFile`, and install the workspace chart in every allowlisted namespace. +If you enable `server.drivers.kubernetes.allowDriverConfig` on the gateway, +also set `gateway.allowDriverConfig=true` on every workspace release. This +grants the namespace-scoped PVC metadata read needed to admit caller-selected +PVC mounts. Leave both values disabled when callers do not need driver config. + ## Wait for the gateway to be ready ```shell diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index a45da33fea..53bb8ceef0 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -574,26 +574,27 @@ metadata: openshell.ai/sandbox-attachable-workspace: "team-a" ``` -Kubernetes checks PVCs, RuntimeClasses, PriorityClasses, and credential/config -references selected through the OpenShell-owned Pod template. Docker and Podman -check engine volume labels. Read-only mounts still require approval. Namespaces -and operator-selected defaults do not waive checks. Newly provisioned private -storage and bootstrap material use driver ownership checks instead. - -RuntimeClasses, PriorityClasses, and image-pull Secrets are shared operator -infrastructure. They must match every fixed required label, but `${workspace}` -entries do not apply to them. PVCs and Secret or ConfigMap data exposed to a -sandbox must also match the resolved workspace label. Kubernetes API-server and -admission-webhook mutations of the live Pod are trusted cluster-operator behavior -and are not used as Workspace User authorization inputs. - -With Kubernetes admission enabled, configured image-pull Secrets in the gateway -namespace must carry the fixed approval labels. In managed mode OpenShell verifies -that source, creates the workspace namespace, and copies the Secret with gateway -ownership metadata. An existing target Secret is updated only when its ownership -metadata matches the gateway and workspace. Operator mode still requires the -approved Secret in each operator-managed namespace. Automatic service-account -tokens and projected token volumes are prohibited. +Kubernetes checks caller-selected PVCs, RuntimeClasses, and PriorityClasses +through the OpenShell-owned Pod template. Docker and Podman check engine volume +labels. Read-only mounts still require approval. Namespaces do not grant +approval. Newly provisioned private storage and bootstrap material use driver +ownership checks instead. + +RuntimeClasses and PriorityClasses are shared operator infrastructure. They must +match every fixed required label, but `${workspace}` entries do not apply to +them. Kubernetes API-server and admission-webhook mutations of the live Pod are +trusted cluster-operator behavior and are not used as Workspace User +authorization inputs. + +Image-pull Secrets are selected by gateway configuration, not by sandbox +callers, and do not participate in resource admission. In managed mode OpenShell +copies each explicitly configured Secret into the workspace namespace with +gateway ownership metadata. An existing target Secret is updated only when its +ownership metadata matches the gateway and workspace. In shared and operator +modes, the configured Secret must already exist in the sandbox namespace. The +typed Kubernetes driver config does not support caller-selected Secret or +ConfigMap volumes or environment references. Automatic service-account tokens +and projected token volumes are prohibited. GPU device attachments are temporarily exempt from label admission. Existing GPU validation still applies. Explicit GPU device settings inside driver JSON @@ -650,9 +651,10 @@ calls until gateway and driver configuration agree again. Helm exposes `server.drivers.kubernetes.allowDriverConfig`, `resourceAdmission.enabled`, and `resourceAdmission.requiredLabels`. The label map defaults to `null`, which uses code defaults without merging those keys -into a custom map. Admission reads need Kubernetes `get` permissions even -though only metadata is requested. Never mark gateway database storage as -attachable. +into a custom map. The Helm chart grants PVC metadata reads only when +`allowDriverConfig` is enabled. Split workspace releases must set the matching +`gateway.allowDriverConfig=true` workspace-chart value. Never mark gateway +database storage as attachable. Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index e331c605e9..aea5c02607 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -94,14 +94,13 @@ Before debugging the compute platform, inspect gateway logs for failures in depe For resource-admission failures, distinguish disabled caller driver config from missing resource approval. Helm defaults `server.drivers.kubernetes.allowDriverConfig` -to false and `resourceAdmission.enabled` to true. Existing PVCs, Secrets, -ConfigMaps, RuntimeClasses, and PriorityClasses need matching administrator-owned -labels; namespace membership and read-only access do not grant approval. Inspect -metadata only when diagnosing credentials. GPU devices do not need labels. -RuntimeClasses, PriorityClasses, and image-pull Secrets match fixed approval -labels but do not use the workspace placeholder. In managed mode, inspect the -approved source image-pull Secret in the gateway namespace and the gateway-owned -copy in the workspace namespace. Legacy workloads without admission provenance need recreation. Do not +to false and `resourceAdmission.enabled` to true. Existing PVCs, RuntimeClasses, +and PriorityClasses need matching administrator-owned labels; namespace +membership and read-only access do not grant approval. GPU devices and +operator-selected image-pull Secrets do not need admission labels. In managed +mode, inspect the configured source image-pull Secret in the gateway namespace +and the gateway-owned copy in the workspace namespace. Legacy workloads without +admission provenance need recreation. Do not automatically label control-plane resources or disable enforcement as a repair. For out-of-tree compute drivers, also check that their versioned admission-policy From df88bedb31300172d9b9516f5fa15250f822a3ea Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 23 Sep 2026 00:30:29 +0000 Subject: [PATCH 9/9] fix(podman): support rootless user namespace configurations (#3527) * test(podman): cover user namespace configurations Signed-off-by: Evan Lezar * fix(podman): support keep-id runtime groups Signed-off-by: Evan Lezar refactor(podman): generalize keep-id group handling Signed-off-by: Evan Lezar * ci(podman): run driver integration tests Signed-off-by: Evan Lezar --------- Signed-off-by: Evan Lezar --- .github/workflows/branch-e2e.yml | 31 +- crates/openshell-driver-podman/src/driver.rs | 4 + .../openshell-driver-podman/src/isolation.rs | 33 +- crates/openshell-sandbox-backend/src/lib.rs | 5 + .../openshell-sandbox/src/boundary_server.rs | 35 +- .../podman/default-userns-baseline.yaml | 113 +++++ .../playbooks/drivers/podman/tests.yaml | 160 +++++++ .../playbooks/drivers/podman/userns-auto.yaml | 9 + .../drivers/podman/userns-keep-id.yaml | 9 + .../drivers/podman/userns-private.yaml | 19 + .../drivers/podman/userns-profile.yaml | 99 +++++ tests/artifacts.nix | 12 +- tests/config.nix | 22 + tests/suites/drivers/Cargo.lock | 393 ++++++++++++++++++ tests/suites/drivers/Cargo.toml | 6 + tests/suites/drivers/podman/Cargo.toml | 15 + .../drivers/podman/fixtures/userns-auto.toml | 4 + .../podman/fixtures/userns-keep-id.toml | 4 + .../podman/fixtures/userns-private.toml | 6 + .../drivers/podman/tests/default_userns.rs | 101 +++++ .../drivers/podman/tests/support/mod.rs | 64 +++ 21 files changed, 1126 insertions(+), 18 deletions(-) create mode 100644 tests/ansible/playbooks/drivers/podman/default-userns-baseline.yaml create mode 100644 tests/ansible/playbooks/drivers/podman/tests.yaml create mode 100644 tests/ansible/playbooks/drivers/podman/userns-auto.yaml create mode 100644 tests/ansible/playbooks/drivers/podman/userns-keep-id.yaml create mode 100644 tests/ansible/playbooks/drivers/podman/userns-private.yaml create mode 100644 tests/ansible/playbooks/drivers/podman/userns-profile.yaml create mode 100644 tests/suites/drivers/Cargo.lock create mode 100644 tests/suites/drivers/Cargo.toml create mode 100644 tests/suites/drivers/podman/Cargo.toml create mode 100644 tests/suites/drivers/podman/fixtures/userns-auto.toml create mode 100644 tests/suites/drivers/podman/fixtures/userns-keep-id.toml create mode 100644 tests/suites/drivers/podman/fixtures/userns-private.toml create mode 100644 tests/suites/drivers/podman/tests/default_userns.rs create mode 100644 tests/suites/drivers/podman/tests/support/mod.rs diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 0491fb3136..99c68b9793 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -247,21 +247,21 @@ jobs: ] # Run driver-specific integration tests: - # TODO: This should be added as soon as we have driver-specific tests enabled in tmachine. - # driver-specific: - # needs: prepare-integration - # permissions: - # actions: read - # contents: read - # packages: read - # uses: ./.github/workflows/integration-runner.yml - # with: - # category: driver-specific - # source-sha: ${{ needs.prepare-integration.outputs.source_sha }} - # integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - # test-matrix: >- - # [ - # ] + driver-specific-integration: + needs: prepare-integration + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/integration-runner.yml + with: + category: driver-specific + source-sha: ${{ needs.prepare-integration.outputs.source_sha }} + integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} + test-matrix: >- + [ + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"driver-podman"} + ] docker-e2e: needs: [pr_metadata, build-binaries, build-images] @@ -451,6 +451,7 @@ jobs: - pr_metadata - conformance-integration - feature-specific-integration + - driver-specific-integration - docker-e2e - vm-e2e - docker-external-driver-e2e diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 335a9ef2e3..4493192b2e 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1159,6 +1159,9 @@ impl PodmanComputeDriver { &workload_id, &uuid::Uuid::new_v4().to_string(), &identity, + crate::isolation::userns_preserves_host_groups( + self.config.userns.as_deref(), + ), child_env, &launch_authentication, )?; @@ -1501,6 +1504,7 @@ impl PodmanComputeDriver { &container_id, generation.as_str(), &restart_metadata.workload_identity, + crate::isolation::userns_preserves_host_groups(self.config.userns.as_deref()), restart_metadata.child_env, &launch_authentication, )?; diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 36b9be92f9..421ae32757 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -13,6 +13,7 @@ use openshell_core::proto::compute::v1::DriverSandbox; use openshell_isolation_interface::contract::{ OuterFenceGuarantee, OuterFenceGuarantees, ResolvedWorkloadIdentity, }; +use openshell_sandbox_backend::ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM; use openshell_sandbox_backend::boundary_protocol::{ BoundaryConfig, BoundaryListener, GatewayVerificationKey, SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, @@ -70,6 +71,12 @@ pub fn channel_volume_name(id: &str) -> String { format!("openshell-channel-{id}") } +/// `keep-id` may retain the gateway user's supplementary groups in the +/// container. Other user-namespace modes, including `auto`, do not. +pub fn userns_preserves_host_groups(userns: Option<&str>) -> bool { + userns.is_some_and(|mode| mode.split(':').next() == Some("keep-id")) +} + fn invalid(error: impl std::fmt::Display) -> ComputeDriverError { ComputeDriverError::Precondition(error.to_string()) } @@ -195,19 +202,26 @@ pub fn bootstrap_archives( container_id: &str, generation: &str, identity: &ResolvedWorkloadIdentity, + allow_extra_supplementary_groups: bool, child_env: HashMap, launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, ) -> Result { launch_authentication.validate().map_err(invalid)?; let session_id = launch_authentication.supervisor.session_id; let tls = generate_sandbox_tls_material(session_id).map_err(invalid)?; - let resource_claims = BTreeMap::from([ + let mut resource_claims = BTreeMap::from([ ("podman.container_id".into(), container_id.into()), ( "podman.image_identity".into(), identity.resource_digest.clone(), ), ]); + if allow_extra_supplementary_groups { + resource_claims.insert( + ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM.into(), + "true".into(), + ); + } let runtime_generation = launch_authentication .supervisor .runtime_generation @@ -496,6 +510,7 @@ mod tests { "container", "generation-1", &identity, + false, child_env.clone(), &authentication, ) @@ -541,6 +556,11 @@ mod tests { .outer_fence .validate(&runtime_descriptor.generation) .unwrap(); + assert!( + !config + .resource_claims + .contains_key(ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM) + ); let restart_metadata: RestartMetadata = serde_json::from_slice( supervisor .get(&PathBuf::from( @@ -558,4 +578,15 @@ mod tests { .any(|window| window == b"PRIVATE KEY") ); } + + #[test] + fn keep_id_is_the_only_userns_mode_that_preserves_host_groups() { + assert!(userns_preserves_host_groups(Some("keep-id"))); + assert!(userns_preserves_host_groups(Some( + "keep-id:uid=1000,gid=1000" + ))); + assert!(!userns_preserves_host_groups(Some("auto"))); + assert!(!userns_preserves_host_groups(Some("private"))); + assert!(!userns_preserves_host_groups(None)); + } } diff --git a/crates/openshell-sandbox-backend/src/lib.rs b/crates/openshell-sandbox-backend/src/lib.rs index 053c57fa4d..87b25dccd3 100644 --- a/crates/openshell-sandbox-backend/src/lib.rs +++ b/crates/openshell-sandbox-backend/src/lib.rs @@ -21,6 +21,11 @@ pub const BACKEND_NAME: &str = "openshell-sandbox"; /// Resource claim set by compute drivers when the workload requests GPU access. pub const GPU_RESOURCE_CLAIM: &str = "openshell.gpu"; +/// Resource claim set when the runtime may retain supplementary groups in +/// addition to the image-derived workload identity. +pub const ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM: &str = + "openshell.identity.allow_extra_supplementary_groups"; + /// Memory-backed parent used for supervisor CA material. pub const SUPERVISOR_CA_RUNTIME_ROOT: &str = "/run/openshell-supervisor-ca"; diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 8fbb79b888..8e33c316c2 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -39,7 +39,6 @@ mod linux { BoundaryConfirmation, BoundaryExec, BoundaryLoopbackConnector, BoundaryProcess, BoundaryTerminal, ExecSession, LoopbackTarget, ResolvedWorkloadIdentity, }; - use openshell_sandbox_backend::GPU_RESOURCE_CLAIM; use openshell_sandbox_backend::mediation::{ self, DnsQueryWire, MediationFrame, MediationFrameKind, }; @@ -53,6 +52,9 @@ mod linux { SandboxConnectionId, SandboxConnectionRegistry, SandboxProtocolAuthenticator, SandboxProtocolPrincipal, }; + use openshell_sandbox_backend::{ + ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM, GPU_RESOURCE_CLAIM, + }; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio_stream::wrappers::ReceiverStream; @@ -389,6 +391,10 @@ mod linux { .resource_claims .get(GPU_RESOURCE_CLAIM) .is_some_and(|value| value == "true") + || config + .resource_claims + .get(ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM) + .is_some_and(|value| value == "true") } fn supplementary_groups_match(actual: &[u32], expected: &[u32], allow_extra: bool) -> bool { @@ -3834,6 +3840,33 @@ mod linux { assert!(!supplementary_groups_match(&[44, 992], &[1001], true)); } + #[test] + fn generic_identity_claim_allows_runtime_supplementary_groups() { + let config = BoundaryConfig { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_id: test_session_id(), + session_rotation: openshell_core::jwt::SessionRotation::new(1) + .expect("session rotation"), + auth_epoch: CredentialEpoch::new(1).expect("auth epoch"), + gateway_id: "test-gateway".to_string(), + verification_keys: vec![], + listener: BoundaryListenerConfig::Vsock { + control_port: 5500, + tls: placeholder_server_tls(), + }, + resource_claims: std::collections::BTreeMap::from([( + ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM.to_string(), + "true".to_string(), + )]), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + outer_fence: test_outer_fence(), + child_env: std::collections::HashMap::new(), + }; + assert!(allows_runtime_supplementary_groups(&config)); + } + #[test] fn control_connection_slots_bound_authenticated_sessions() { let active = Arc::new(AtomicUsize::new(MAX_CONTROL_CONNECTIONS - 1)); diff --git a/tests/ansible/playbooks/drivers/podman/default-userns-baseline.yaml b/tests/ansible/playbooks/drivers/podman/default-userns-baseline.yaml new file mode 100644 index 0000000000..4980bebdc4 --- /dev/null +++ b/tests/ansible/playbooks/drivers/podman/default-userns-baseline.yaml @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Capture the default rootless Podman user-namespace mapping + hosts: all + gather_facts: false + vars: + podman_reference_image: "{{ openshell_podman_reference_image }}" + podman_reference_uid_map: /var/lib/openshell-test-inputs/podman/reference-uid-map + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Read OpenShell gateway service user + become: true + ansible.builtin.command: + argv: + - systemctl + - show + - openshell-gateway.service + - --property + - User + - --value + changed_when: false + register: openshell_gateway_service_user + + - name: Require the rootless Podman gateway user + ansible.builtin.assert: + that: + - openshell_gateway_service_user.stdout == "tmachine" + fail_msg: >- + The Podman default-userns suite requires a rootless Podman gateway + running as tmachine, not {{ openshell_gateway_service_user.stdout | default("unknown") }} + + - name: Read OpenShell gateway configuration + become: true + ansible.builtin.slurp: + src: /etc/openshell/gateway.toml + register: openshell_gateway_config + + - name: Require unconfigured Podman user namespaces + ansible.builtin.assert: + that: + - >- + (openshell_gateway_config.content | b64decode) + is not regex('(?m)^\\s*(userns|uidmap|gidmap)\\s*=') + fail_msg: >- + The Podman default-userns suite requires gateway.toml to omit userns, + uidmap, and gidmap. + + - name: Resolve tmachine UID + ansible.builtin.command: + argv: + - id + - -u + - tmachine + changed_when: false + register: tmachine_uid + + - name: Create Podman test-input directory + become: true + ansible.builtin.file: + path: "{{ podman_reference_uid_map | dirname }}" + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Pull the Podman reference image + become: true + become_user: tmachine + ansible.builtin.command: + argv: [podman, pull, "{{ podman_reference_image }}"] + environment: + HOME: /home/tmachine + XDG_RUNTIME_DIR: "/run/user/{{ tmachine_uid.stdout }}" + changed_when: false + + - name: Capture direct Podman default UID mapping + become: true + become_user: tmachine + ansible.builtin.command: + argv: + - podman + - run + - --rm + - --pull + - never + - --entrypoint + - /bin/cat + - "{{ podman_reference_image }}" + - /proc/self/uid_map + environment: + HOME: /home/tmachine + XDG_RUNTIME_DIR: "/run/user/{{ tmachine_uid.stdout }}" + changed_when: false + register: podman_default_uid_map + + - name: Require a direct Podman UID mapping + ansible.builtin.assert: + that: + - podman_default_uid_map.stdout | trim | length > 0 + fail_msg: Direct rootless Podman reference container returned no UID mapping + + - name: Store direct Podman default UID mapping + become: true + ansible.builtin.copy: + content: "{{ podman_default_uid_map.stdout | trim }}\n" + dest: "{{ podman_reference_uid_map }}" + owner: tmachine + group: tmachine + mode: "0600" diff --git a/tests/ansible/playbooks/drivers/podman/tests.yaml b/tests/ansible/playbooks/drivers/podman/tests.yaml new file mode 100644 index 0000000000..abd05bea70 --- /dev/null +++ b/tests/ansible/playbooks/drivers/podman/tests.yaml @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Run Podman archive tests + hosts: all + gather_facts: false + vars: + podman_test_root: /var/lib/openshell-driver-tests/podman + podman_test_input_dir: /var/lib/openshell-test-inputs/podman + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Create Podman test directory + become: true + ansible.builtin.file: + path: "{{ podman_test_root }}" + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Install Podman test bundle + become: true + ansible.builtin.unarchive: + src: "{{ openshell_podman_test_bundle }}" + dest: "{{ podman_test_root }}" + owner: tmachine + group: tmachine + + - name: Remove any previous OpenShell gateway registration + ansible.builtin.command: + argv: [/usr/local/bin/openshell, gateway, remove, tmachine] + changed_when: false + failed_when: false + + - name: Register the configured OpenShell gateway + ansible.builtin.command: + argv: [/usr/local/bin/openshell, gateway, add, http://127.0.0.1:17670, --local, --name, tmachine] + + - name: Run Podman archive tests + ansible.builtin.command: + argv: [cargo-nextest, nextest, run, --archive-file, "{{ podman_test_root }}/tests.tar.zst", --workspace-remap, "{{ podman_test_root }}", --no-capture] + environment: + OPENSHELL_BIN: /usr/local/bin/openshell + OPENSHELL_TEST_INPUT_DIR: "{{ podman_test_input_dir }}" + OPENSHELL_PODMAN_TEST_IMAGE: "{{ openshell_podman_test_image | default('') }}" + register: podman_test_result + changed_when: false + failed_when: false + + - name: Show Podman test diagnostics + ansible.builtin.debug: + var: podman_test_result.stderr_lines + + - name: Show Podman test result + ansible.builtin.debug: + var: podman_test_result.stdout_lines + + - name: Capture OpenShell gateway journal after Podman test failure + become: true + ansible.builtin.command: + argv: + - journalctl + - --unit + - openshell-gateway.service + - --no-pager + - --lines + - "200" + changed_when: false + failed_when: false + when: podman_test_result.rc != 0 + register: podman_gateway_journal + + - name: Show OpenShell gateway journal after Podman test failure + ansible.builtin.debug: + var: podman_gateway_journal.stdout_lines + when: podman_test_result.rc != 0 + + - name: Resolve tmachine UID after Podman test failure + ansible.builtin.command: + argv: [id, --user, tmachine] + changed_when: false + when: podman_test_result.rc != 0 + register: podman_tmachine_uid + + - name: Discover Podman containers after test failure + become: true + become_user: tmachine + ansible.builtin.command: + argv: [timeout, "15", podman, ps, --all, --quiet, --no-trunc] + environment: + HOME: /home/tmachine + XDG_RUNTIME_DIR: "/run/user/{{ podman_tmachine_uid.stdout }}" + changed_when: false + failed_when: false + when: podman_test_result.rc != 0 + register: podman_failed_containers + + - name: Capture rootless Podman journal after test failure + become: true + become_user: tmachine + ansible.builtin.command: + argv: [journalctl, --user, --unit, podman.service, --no-pager, --lines, "200"] + environment: + HOME: /home/tmachine + XDG_RUNTIME_DIR: "/run/user/{{ podman_tmachine_uid.stdout }}" + changed_when: false + failed_when: false + when: podman_test_result.rc != 0 + register: podman_service_journal + + - name: Show rootless Podman journal after test failure + ansible.builtin.debug: + var: podman_service_journal.stdout_lines + when: podman_test_result.rc != 0 + + - name: Capture Podman container inspection after test failure + become: true + become_user: tmachine + ansible.builtin.command: + argv: [timeout, "15", podman, inspect, "{{ item }}"] + environment: + HOME: /home/tmachine + XDG_RUNTIME_DIR: "/run/user/{{ podman_tmachine_uid.stdout }}" + changed_when: false + failed_when: false + when: podman_test_result.rc != 0 + loop: "{{ podman_failed_containers.stdout_lines | default([]) }}" + register: podman_failed_container_inspection + + - name: Show Podman container inspection after test failure + ansible.builtin.debug: + var: podman_failed_container_inspection.results + when: podman_test_result.rc != 0 + + - name: Capture Podman container logs after test failure + become: true + become_user: tmachine + ansible.builtin.command: + argv: [timeout, "15", podman, logs, "{{ item }}"] + environment: + HOME: /home/tmachine + XDG_RUNTIME_DIR: "/run/user/{{ podman_tmachine_uid.stdout }}" + changed_when: false + failed_when: false + when: podman_test_result.rc != 0 + loop: "{{ podman_failed_containers.stdout_lines | default([]) }}" + register: podman_failed_container_logs + + - name: Show Podman container logs after test failure + ansible.builtin.debug: + var: podman_failed_container_logs.results + when: podman_test_result.rc != 0 + + - name: Require Podman archive success + ansible.builtin.assert: + that: [podman_test_result.rc == 0] + fail_msg: Podman archive tests failed diff --git a/tests/ansible/playbooks/drivers/podman/userns-auto.yaml b/tests/ansible/playbooks/drivers/podman/userns-auto.yaml new file mode 100644 index 0000000000..353f902c2c --- /dev/null +++ b/tests/ansible/playbooks/drivers/podman/userns-auto.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- import_playbook: userns-profile.yaml + vars: + podman_userns_profile: auto + podman_userns_config: "{{ openshell_podman_userns_auto_config }}" + podman_userns_reference_args: [--userns, auto] diff --git a/tests/ansible/playbooks/drivers/podman/userns-keep-id.yaml b/tests/ansible/playbooks/drivers/podman/userns-keep-id.yaml new file mode 100644 index 0000000000..e7ec08cce4 --- /dev/null +++ b/tests/ansible/playbooks/drivers/podman/userns-keep-id.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- import_playbook: userns-profile.yaml + vars: + podman_userns_profile: keep-id + podman_userns_config: "{{ openshell_podman_userns_keep_id_config }}" + podman_userns_reference_args: [--userns, keep-id] diff --git a/tests/ansible/playbooks/drivers/podman/userns-private.yaml b/tests/ansible/playbooks/drivers/podman/userns-private.yaml new file mode 100644 index 0000000000..d24c0f38bc --- /dev/null +++ b/tests/ansible/playbooks/drivers/podman/userns-private.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- import_playbook: userns-profile.yaml + vars: + podman_userns_profile: private + podman_userns_config: "{{ openshell_podman_userns_private_config }}" + podman_userns_reference_args: + # Podman infers a private namespace from explicit maps; its CLI rejects + # combining --userns private with --uidmap/--gidmap. + - --uidmap + - 0:0:1 + - --uidmap + - 1:1:65535 + - --gidmap + - 0:0:1 + - --gidmap + - 1:1:65535 diff --git a/tests/ansible/playbooks/drivers/podman/userns-profile.yaml b/tests/ansible/playbooks/drivers/podman/userns-profile.yaml new file mode 100644 index 0000000000..ba5efc0f59 --- /dev/null +++ b/tests/ansible/playbooks/drivers/podman/userns-profile.yaml @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Configure and capture a Podman user-namespace reference + hosts: all + gather_facts: false + vars: + podman_reference_image: "{{ openshell_podman_reference_image }}" + podman_reference_uid_map: /var/lib/openshell-test-inputs/podman/reference-uid-map + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Require the rootless Podman gateway user + become: true + ansible.builtin.command: + argv: + - systemctl + - show + - openshell-gateway.service + - --property + - User + - --value + changed_when: false + register: openshell_gateway_service_user + + - name: Assert the rootless Podman gateway user + ansible.builtin.assert: + that: + - openshell_gateway_service_user.stdout == "tmachine" + fail_msg: >- + The Podman {{ podman_userns_profile }} suite requires a rootless Podman gateway + running as tmachine, not {{ openshell_gateway_service_user.stdout | default("unknown") }} + + - name: Apply the Podman user-namespace fixture + become: true + ansible.builtin.blockinfile: + path: /etc/openshell/gateway.toml + marker: "# {mark} OpenShell Podman userns test fixture" + block: "{{ lookup('ansible.builtin.file', podman_userns_config) | trim }}" + + - name: Restart OpenShell gateway with the Podman user-namespace fixture + become: true + ansible.builtin.systemd_service: + name: openshell-gateway.service + state: restarted + + - name: Wait for the configured OpenShell gateway + ansible.builtin.wait_for: + host: 127.0.0.1 + port: 17670 + timeout: 60 + + - name: Resolve tmachine UID + ansible.builtin.command: + argv: + - id + - -u + - tmachine + changed_when: false + register: tmachine_uid + + - name: Capture direct Podman user-namespace mapping + become: true + become_user: tmachine + ansible.builtin.command: + argv: "{{ [\"podman\", \"run\", \"--rm\", \"--pull\", \"never\"] + podman_userns_reference_args + [\"--entrypoint\", \"/bin/cat\", podman_reference_image, \"/proc/self/uid_map\"] }}" + environment: + HOME: /home/tmachine + XDG_RUNTIME_DIR: "/run/user/{{ tmachine_uid.stdout }}" + changed_when: false + register: podman_userns_uid_map + + - name: Require a direct Podman user-namespace mapping + ansible.builtin.assert: + that: + - podman_userns_uid_map.stdout | trim | length > 0 + fail_msg: >- + Direct rootless Podman {{ podman_userns_profile }} reference container + returned no UID mapping + + - name: Create Podman test-input directory + become: true + ansible.builtin.file: + path: "{{ podman_reference_uid_map | dirname }}" + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Store direct Podman user-namespace mapping + become: true + ansible.builtin.copy: + content: "{{ podman_userns_uid_map.stdout | trim }}\n" + dest: "{{ podman_reference_uid_map }}" + owner: tmachine + group: tmachine + mode: "0600" diff --git a/tests/artifacts.nix b/tests/artifacts.nix index e44d1751e6..9b1d0c6dfb 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -92,9 +92,17 @@ let target = muslToolchain.target; output = "artifacts/test-archives/${muslToolchain.target}/provider-refresh-keycloak-tests.tar"; }; + podmanDriverArchive = mkTestArchive { + name = "podman-driver"; + workspacePath = "tests/suites/drivers"; + manifestPath = "tests/suites/drivers/Cargo.toml"; + package = "openshell-test-suite-podman"; + target = muslToolchain.target; + output = "artifacts/test-archives/${muslToolchain.target}/openshell-podman-tests.tar"; + }; in rec { - inherit conformanceCliArchive providerRefreshKeycloakArchive; + inherit conformanceCliArchive providerRefreshKeycloakArchive podmanDriverArchive; binaries = pkgs.writeShellApplication { name = "build-artifacts-binaries"; @@ -139,10 +147,12 @@ rec { runtimeInputs = [ conformanceCliArchive providerRefreshKeycloakArchive + podmanDriverArchive ]; text = '' build-openshell-conformance-test-archive build-provider-refresh-keycloak-test-archive + build-podman-driver-test-archive ''; }; diff --git a/tests/config.nix b/tests/config.nix index c8d80e6ebe..4b5d5b7d96 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -126,6 +126,28 @@ let provider_refresh_keycloak_test_bundle = "../artifacts/test-archives/${muslTarget}/provider-refresh-keycloak-tests.tar"; }; } + { + name = "driver-podman"; + playbooks = [ + "ansible/playbooks/drivers/podman/default-userns-baseline.yaml" + "ansible/playbooks/drivers/podman/tests.yaml" + "ansible/playbooks/drivers/podman/userns-auto.yaml" + "ansible/playbooks/drivers/podman/tests.yaml" + "ansible/playbooks/drivers/podman/userns-keep-id.yaml" + "ansible/playbooks/drivers/podman/tests.yaml" + "ansible/playbooks/drivers/podman/userns-private.yaml" + "ansible/playbooks/drivers/podman/tests.yaml" + ]; + inputs = { + openshell_podman_test_bundle = "../artifacts/test-archives/${muslTarget}/openshell-podman-tests.tar"; + # Match OpenShell's compiled-in default so direct Podman and + # OpenShell containers resolve the same workload image metadata. + openshell_podman_reference_image = "nvcr.io/nvidia/base/ubuntu:24.04"; + openshell_podman_userns_auto_config = "suites/drivers/podman/fixtures/userns-auto.toml"; + openshell_podman_userns_keep_id_config = "suites/drivers/podman/fixtures/userns-keep-id.toml"; + openshell_podman_userns_private_config = "suites/drivers/podman/fixtures/userns-private.toml"; + }; + } ]; }; diff --git a/tests/suites/drivers/Cargo.lock b/tests/suites/drivers/Cargo.lock new file mode 100644 index 0000000000..ad7a993fa9 --- /dev/null +++ b/tests/suites/drivers/Cargo.lock @@ -0,0 +1,393 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "openshell-conformance" +version = "0.0.0" +dependencies = [ + "rand", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "openshell-test-suite-podman" +version = "0.0.0" +dependencies = [ + "openshell-conformance", + "serde", + "tokio", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tests/suites/drivers/Cargo.toml b/tests/suites/drivers/Cargo.toml new file mode 100644 index 0000000000..0ec76752b2 --- /dev/null +++ b/tests/suites/drivers/Cargo.toml @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[workspace] +resolver = "2" +members = ["podman"] diff --git a/tests/suites/drivers/podman/Cargo.toml b/tests/suites/drivers/podman/Cargo.toml new file mode 100644 index 0000000000..1a06b69e8d --- /dev/null +++ b/tests/suites/drivers/podman/Cargo.toml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-test-suite-podman" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" + +[dependencies] +openshell-conformance = { path = "../../../../crates/openshell-conformance" } +serde = { version = "1", features = ["derive"] } +tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/suites/drivers/podman/fixtures/userns-auto.toml b/tests/suites/drivers/podman/fixtures/userns-auto.toml new file mode 100644 index 0000000000..3701222828 --- /dev/null +++ b/tests/suites/drivers/podman/fixtures/userns-auto.toml @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +userns = "auto" diff --git a/tests/suites/drivers/podman/fixtures/userns-keep-id.toml b/tests/suites/drivers/podman/fixtures/userns-keep-id.toml new file mode 100644 index 0000000000..c52cdb528f --- /dev/null +++ b/tests/suites/drivers/podman/fixtures/userns-keep-id.toml @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +userns = "keep-id" diff --git a/tests/suites/drivers/podman/fixtures/userns-private.toml b/tests/suites/drivers/podman/fixtures/userns-private.toml new file mode 100644 index 0000000000..48a7968d13 --- /dev/null +++ b/tests/suites/drivers/podman/fixtures/userns-private.toml @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +userns = "private" +uidmap = ["0:0:1", "1:1:65535"] +gidmap = ["0:0:1", "1:1:65535"] diff --git a/tests/suites/drivers/podman/tests/default_userns.rs b/tests/suites/drivers/podman/tests/default_userns.rs new file mode 100644 index 0000000000..7618e643af --- /dev/null +++ b/tests/suites/drivers/podman/tests/default_userns.rs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman-driver user-namespace integration tests. + +mod support; + +use openshell_conformance::OpenShellRunner; +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use support::assert_podman_gateway; + +const SANDBOX_TIMEOUT: Duration = Duration::from_secs(300); +const PODMAN_TEST_INPUT_DIR_ENV: &str = "OPENSHELL_TEST_INPUT_DIR"; +const PODMAN_TEST_IMAGE_ENV: &str = "OPENSHELL_PODMAN_TEST_IMAGE"; + +/// Verify that the gateway's user-namespace configuration matches Podman's +/// direct behavior for the same profile. +/// +/// The test runs a short-lived sandbox command and compares its user-namespace +/// mapping with the direct-Podman reference stored at +/// `OPENSHELL_TEST_INPUT_DIR/reference-uid-map`. The tmachine pre-test +/// playbook creates that reference in the same gateway-user context. This deliberately +/// avoids baking a particular Podman mapping into OpenShell's test contract. +/// +#[tokio::test] +async fn configured_userns_matches_podman_reference() { + let mut runner = OpenShellRunner::from_env("podman-userns") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + assert_podman_gateway(&runner).await?; + + let test_input_dir = std::env::var_os(PODMAN_TEST_INPUT_DIR_ENV) + .map(PathBuf::from) + .ok_or_else(|| format!("{PODMAN_TEST_INPUT_DIR_ENV} must name the Podman test-input directory"))?; + let expected_path = test_input_dir.join("reference-uid-map"); + let expected_uid_map = fs::read_to_string(&expected_path).map_err(|error| { + format!( + "could not read Podman reference UID map {}: {error}", + expected_path.display() + ) + })?; + let expected_uid_map = normalize_uid_map(&expected_uid_map).ok_or_else(|| { + format!( + "Podman reference UID map {} contains no mappings", + expected_path.display() + ) + })?; + + let workload_image = std::env::var(PODMAN_TEST_IMAGE_ENV) + .ok() + .filter(|image| !image.trim().is_empty()); + let sandbox_name = format!("pu-{}", runner.id()); + runner.track_sandbox(&sandbox_name); + let mut create_args = vec!["sandbox", "create", "--name", &sandbox_name]; + if let Some(image) = workload_image.as_deref() { + create_args.extend(["--from", image]); + } + create_args.extend(["--no-tty", "--", "cat", "/proc/self/uid_map"]); + let run = runner + .step("userns/uid-map") + .description("sandbox exposes its UID map") + .with_timeout(SANDBOX_TIMEOUT) + .run(&create_args) + .await + .map_err(|error| error.to_string())?; + run.require_success()?; + let sandbox_uid_map = normalize_uid_map(run.stdout()).ok_or_else(|| { + run.failure_diagnostic("sandbox returns a non-empty UID map") + })?; + if sandbox_uid_map != expected_uid_map { + return Err(format!( + "sandbox UID map differs from the direct Podman reference:\nexpected:\n{expected_uid_map}\nactual:\n{sandbox_uid_map}" + )); + } + Ok(()) + } + .await; + + if let Err(error) = runner.finish(result).await { + panic!("Podman userns test failed:\n{error}"); + } +} + +fn normalize_uid_map(value: &str) -> Option { + let mappings = value + .lines() + .filter_map(|line| { + let fields = line.split_whitespace().collect::>(); + (fields.len() == 3 + && fields + .iter() + .all(|field| field.bytes().all(|byte| byte.is_ascii_digit()))) + .then(|| fields.join(" ")) + }) + .collect::>(); + (!mappings.is_empty()).then(|| mappings.join("\n")) +} diff --git a/tests/suites/drivers/podman/tests/support/mod.rs b/tests/suites/drivers/podman/tests/support/mod.rs new file mode 100644 index 0000000000..5bde19b0c1 --- /dev/null +++ b/tests/suites/drivers/podman/tests/support/mod.rs @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use openshell_conformance::OpenShellRunner; +use serde::Deserialize; +use std::time::Duration; + +#[derive(Debug, Deserialize)] +struct GatewayInfo { + status: String, + compute_drivers: Vec, +} + +#[derive(Debug, Deserialize)] +struct ComputeDriver { + name: String, + capabilities: ComputeDriverCapabilities, +} + +#[derive(Debug, Deserialize)] +struct ComputeDriverCapabilities { + driver_name: String, +} + +/// Require the target gateway to use Podman as its only compute driver. +/// +/// This interrogates the running gateway rather than accepting a runner +/// environment variable. A driver-specific suite must fail, rather than skip, +/// when it is pointed at the wrong gateway. +pub async fn assert_podman_gateway(runner: &OpenShellRunner) -> Result<(), String> { + let result = runner + .step("preflight/driver-podman") + .description("gateway reports Podman as its only compute driver") + .with_timeout(Duration::from_secs(10)) + .run(&["gateway", "info", "--output", "json"]) + .await + .map_err(|error| format!("could not query gateway compute drivers: {error}"))?; + result.require_success()?; + + let info = result + .json::() + .map_err(|error| format!("gateway returned invalid driver information: {error}"))?; + if info.status != "healthy" { + return Err(format!( + "Podman test suite requires a healthy gateway; gateway status is {:?}", + info.status + )); + } + + let driver_names = info + .compute_drivers + .iter() + .map(|driver| driver.name.as_str()) + .collect::>(); + let podman_only = matches!(info.compute_drivers.as_slice(), [driver] + if driver.name == "podman" && driver.capabilities.driver_name == "podman"); + if !podman_only { + return Err(format!( + "Podman test suite requires exactly one Podman compute driver; gateway reported {driver_names:?}" + )); + } + + Ok(()) +}