Skip to content

Commit 2026bdc

Browse files
committed
fix(docker): launch the admitted agent spec and harden the notify boundary
- Deliver the driver's real main-process spec via OPENSHELL_MAIN_PROCESS_SPEC instead of a hardcoded placeholder shell loop, so the sandbox command and tty semantics reach the Docker workload container. - Move the topology payload (which carries the boundary listener token) out of world-readable argv into a driver-owned 0600 file. - Deliver the admitted isolation backend on the driver-owned OPENSHELL_ADMITTED_ISOLATION_BACKEND channel. - Gate the host-supervisor spawn path as Linux-only with a fail-closed stub so the driver compiles on non-Linux gateways again. - Re-validate seccomp notification liveness (SECCOMP_IOCTL_NOTIF_ID_VALID) after every pid-derived read, so a recycled PID cannot attach another process's identity or memory to a mediated connection. - Bound the retained pending-socket map so a socket()/close() loop cannot exhaust host supervisor descriptors; eviction fails closed against the container's disabled network. - Restore Docker default-profile parity for escape-relevant syscalls with an explicit EPERM deny group on top of the notify profile. - Enable TCP_NODELAY on the injected mediation socket pair. - Document host_gateway_ip and the Unsupported error in RFC 0012; drop the unrelated sccache configuration change. Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent f882dcb commit 2026bdc

4 files changed

Lines changed: 204 additions & 21 deletions

File tree

‎crates/openshell-driver-docker/src/isolation.rs‎

Lines changed: 144 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,48 @@ fn resource_key(sandbox_id: &str, launch_generation: &str) -> String {
343343
format!("{:016x}", hasher.finish())
344344
}
345345

346+
/// Syscalls Docker's default profile leaves off its allowlist that matter for
347+
/// containment (module loading, mount manipulation, kernel keyring, kexec,
348+
/// namespace re-entry, memory introspection, and fault-handling takeover).
349+
/// Restricted to names available on both amd64 and arm64 so runc can resolve
350+
/// every entry.
351+
const DENIED_SYSCALLS: &[&str] = &[
352+
"acct",
353+
"add_key",
354+
"clock_adjtime",
355+
"clock_settime",
356+
"delete_module",
357+
"finit_module",
358+
"get_mempolicy",
359+
"init_module",
360+
"kexec_file_load",
361+
"kexec_load",
362+
"keyctl",
363+
"lookup_dcookie",
364+
"mbind",
365+
"mount",
366+
"move_mount",
367+
"move_pages",
368+
"name_to_handle_at",
369+
"open_by_handle_at",
370+
"open_tree",
371+
"perf_event_open",
372+
"pivot_root",
373+
"process_vm_readv",
374+
"process_vm_writev",
375+
"quotactl",
376+
"reboot",
377+
"request_key",
378+
"set_mempolicy",
379+
"setns",
380+
"settimeofday",
381+
"swapoff",
382+
"swapon",
383+
"umount2",
384+
"unshare",
385+
"userfaultfd",
386+
];
387+
346388
fn build_create_body(
347389
spec: &DockerBoundarySpec,
348390
sandbox: &SandboxContext,
@@ -351,13 +393,22 @@ fn build_create_body(
351393
labels: HashMap<String, String>,
352394
proxy_tls_dir: Option<&Path>,
353395
) -> Result<ContainerCreateBody, BackendError> {
396+
// Attaching a notify listener replaces Docker's default allowlist
397+
// profile, so the default action must be ALLOW for the workload to run at
398+
// all. Restore parity for the escape-relevant syscalls Docker's default
399+
// profile denies with an explicit EPERM group; the workload keeps
400+
// cap_drop ALL and no-new-privileges on top of this.
354401
let seccomp = serde_json::json!({
355402
"defaultAction": "SCMP_ACT_ALLOW",
356403
"listenerPath": listener_path,
357404
"listenerMetadata": listener_metadata,
358405
"syscalls": [{
359406
"names": ["socket", "connect", "sendto", "bpf"],
360407
"action": "SCMP_ACT_NOTIFY"
408+
}, {
409+
"names": DENIED_SYSCALLS,
410+
"action": "SCMP_ACT_ERRNO",
411+
"errnoRet": 1
361412
}]
362413
});
363414
let seccomp = serde_json::to_string(&seccomp).map_err(|error| {
@@ -1190,10 +1241,18 @@ const fn ioctl_write<T>(kind: u8, number: u8) -> libc::Ioctl {
11901241

11911242
const SECCOMP_IOCTL_NOTIF_RECV: libc::Ioctl = ioctl_read_write::<SeccompNotif>(b'!', 0);
11921243
const SECCOMP_IOCTL_NOTIF_SEND: libc::Ioctl = ioctl_read_write::<SeccompNotifResp>(b'!', 1);
1244+
const SECCOMP_IOCTL_NOTIF_ID_VALID: libc::Ioctl = ioctl_write::<u64>(b'!', 2);
11931245
const SECCOMP_IOCTL_NOTIF_ADDFD: libc::Ioctl = ioctl_write::<SeccompNotifAddfd>(b'!', 3);
11941246
const SECCOMP_USER_NOTIF_FLAG_CONTINUE: u32 = 1;
11951247
const SECCOMP_ADDFD_FLAG_SEND: u32 = 2;
11961248

1249+
/// Ceiling on retained per-boundary pending sockets. `close()` is not
1250+
/// notified, so without a bound a `socket(); close()` loop would grow the map
1251+
/// (and its retained host descriptors) without limit. Evicting the oldest
1252+
/// entry fails closed: a later `connect` on an evicted descriptor receives
1253+
/// CONTINUE and runs against the container's disabled network.
1254+
const MAX_PENDING_SOCKETS: usize = 512;
1255+
11971256
enum PendingSocket {
11981257
Tcp(std::net::TcpStream),
11991258
Dns {
@@ -1202,8 +1261,62 @@ enum PendingSocket {
12021261
},
12031262
}
12041263

1264+
struct PendingEntry {
1265+
socket: PendingSocket,
1266+
sequence: u64,
1267+
}
1268+
1269+
#[derive(Default)]
1270+
struct PendingSockets {
1271+
entries: HashMap<(u32, i32), PendingEntry>,
1272+
next_sequence: u64,
1273+
}
1274+
1275+
impl PendingSockets {
1276+
fn insert(&mut self, key: (u32, i32), socket: PendingSocket) {
1277+
if self.entries.len() >= MAX_PENDING_SOCKETS
1278+
&& let Some(oldest) = self
1279+
.entries
1280+
.iter()
1281+
.min_by_key(|(_, entry)| entry.sequence)
1282+
.map(|(key, _)| *key)
1283+
{
1284+
self.entries.remove(&oldest);
1285+
}
1286+
let sequence = self.next_sequence;
1287+
self.next_sequence += 1;
1288+
self.entries.insert(key, PendingEntry { socket, sequence });
1289+
}
1290+
1291+
fn get_mut(&mut self, key: (u32, i32)) -> Option<&mut PendingSocket> {
1292+
self.entries.get_mut(&key).map(|entry| &mut entry.socket)
1293+
}
1294+
1295+
fn remove(&mut self, key: (u32, i32)) -> Option<PendingSocket> {
1296+
self.entries.remove(&key).map(|entry| entry.socket)
1297+
}
1298+
}
1299+
1300+
/// Whether the notifying process is still blocked in the intercepted syscall.
1301+
///
1302+
/// Identity and remote memory are read via `notification.pid`; if that process
1303+
/// died and the kernel PID was reused, those reads describe another process.
1304+
/// The kernel invalidates the notification ID on death, so a re-check after
1305+
/// every pid-derived read closes the reuse race.
1306+
fn notification_id_is_valid(listener_fd: &OwnedFd, notification_id: u64) -> bool {
1307+
// SAFETY: request code and payload layout match linux/seccomp.h; the
1308+
// kernel only reads the notification ID.
1309+
unsafe {
1310+
libc::ioctl(
1311+
listener_fd.as_raw_fd(),
1312+
SECCOMP_IOCTL_NOTIF_ID_VALID,
1313+
&notification_id,
1314+
) == 0
1315+
}
1316+
}
1317+
12051318
fn run_notification_worker(listener_fd: OwnedFd, mediation: Arc<DockerMediation>) {
1206-
let mut pending = HashMap::<(u32, i32), PendingSocket>::new();
1319+
let mut pending = PendingSockets::default();
12071320
loop {
12081321
let mut notification = SeccompNotif::default();
12091322
// SAFETY: the request code and C layout match linux/seccomp.h, and the
@@ -1252,7 +1365,7 @@ fn run_notification_worker(listener_fd: OwnedFd, mediation: Arc<DockerMediation>
12521365
fn handle_socket_notification(
12531366
listener_fd: &OwnedFd,
12541367
notification: &SeccompNotif,
1255-
pending: &mut HashMap<(u32, i32), PendingSocket>,
1368+
pending: &mut PendingSockets,
12561369
) -> Result<(), BackendError> {
12571370
let domain = i32::try_from(notification.data.args[0]).unwrap_or_default();
12581371
let socket_type = i32::try_from(notification.data.args[1]).unwrap_or_default();
@@ -1292,6 +1405,10 @@ fn handle_socket_notification(
12921405
BackendError::Process(format!("configure mediated TCP endpoint: {error}"))
12931406
})?;
12941407
}
1408+
// Loopback still pays the delayed-ACK timer; keep small mediated
1409+
// frames from stalling on Nagle.
1410+
let _ = injected.set_nodelay(true);
1411+
let _ = peer.set_nodelay(true);
12951412
let remote_fd =
12961413
inject_fd_and_respond(listener_fd, notification.id, injected.as_raw_fd(), cloexec)?;
12971414
(remote_fd, PendingSocket::Tcp(peer))
@@ -1324,12 +1441,12 @@ fn handle_socket_notification(
13241441
fn handle_connect_notification(
13251442
listener_fd: &OwnedFd,
13261443
notification: &SeccompNotif,
1327-
pending: &mut HashMap<(u32, i32), PendingSocket>,
1444+
pending: &mut PendingSockets,
13281445
mediation: &DockerMediation,
13291446
) -> Result<(), BackendError> {
13301447
let remote_fd = i32::try_from(notification.data.args[0]).unwrap_or(-1);
13311448
let key = (notification.pid, remote_fd);
1332-
let Some(socket) = pending.get_mut(&key) else {
1449+
let Some(socket) = pending.get_mut(key) else {
13331450
return send_notification_response(
13341451
listener_fd,
13351452
notification.id,
@@ -1344,9 +1461,15 @@ fn handle_connect_notification(
13441461
notification.data.args[2],
13451462
)?;
13461463
let identity = resolve_process_identity(notification.pid);
1464+
// The sockaddr and identity above were read through `notification.pid`;
1465+
// deny the syscall if the notification is no longer live so a recycled
1466+
// PID cannot attach another process's identity to this connection.
1467+
if !notification_id_is_valid(listener_fd, notification.id) {
1468+
return send_notification_response(listener_fd, notification.id, 0, libc::EPERM, 0);
1469+
}
13471470
match socket {
13481471
PendingSocket::Tcp(_) => {
1349-
let PendingSocket::Tcp(stream) = pending.remove(&key).expect("pending socket exists")
1472+
let PendingSocket::Tcp(stream) = pending.remove(key).expect("pending socket exists")
13501473
else {
13511474
unreachable!();
13521475
};
@@ -1380,12 +1503,12 @@ fn handle_connect_notification(
13801503
fn handle_sendto_notification(
13811504
listener_fd: &OwnedFd,
13821505
notification: &SeccompNotif,
1383-
pending: &mut HashMap<(u32, i32), PendingSocket>,
1506+
pending: &mut PendingSockets,
13841507
mediation: &DockerMediation,
13851508
) -> Result<(), BackendError> {
13861509
let remote_fd = i32::try_from(notification.data.args[0]).unwrap_or(-1);
13871510
let Some(PendingSocket::Dns { injected, peer }) =
1388-
pending.get_mut(&(notification.pid, remote_fd))
1511+
pending.get_mut((notification.pid, remote_fd))
13891512
else {
13901513
return send_notification_response(
13911514
listener_fd,
@@ -1407,17 +1530,26 @@ fn handle_sendto_notification(
14071530
} else if peer.is_some() {
14081531
return send_notification_response(listener_fd, notification.id, 0, libc::ENOTCONN, 0);
14091532
}
1533+
let session_identity = peer
1534+
.is_some()
1535+
.then(|| resolve_process_identity(notification.pid));
1536+
let length = usize::try_from(notification.data.args[2])
1537+
.unwrap_or(usize::MAX)
1538+
.min(8 * 1024);
1539+
let request = read_remote_bytes(notification.pid, notification.data.args[1], length)?;
1540+
// The identity and request bytes above were read through
1541+
// `notification.pid`; deny the syscall if the notification is no longer
1542+
// live so a recycled PID cannot forward another process's data.
1543+
if !notification_id_is_valid(listener_fd, notification.id) {
1544+
return send_notification_response(listener_fd, notification.id, 0, libc::EPERM, 0);
1545+
}
14101546
if let Some(peer) = peer.take() {
14111547
spawn_dns_udp_session(
14121548
peer,
1413-
resolve_process_identity(notification.pid),
1549+
session_identity.expect("identity resolved while the peer was retained"),
14141550
mediation.dns_tx.clone(),
14151551
);
14161552
}
1417-
let length = usize::try_from(notification.data.args[2])
1418-
.unwrap_or(usize::MAX)
1419-
.min(8 * 1024);
1420-
let request = read_remote_bytes(notification.pid, notification.data.args[1], length)?;
14211553
let written = injected
14221554
.send(&request)
14231555
.map_err(|error| BackendError::Process(format!("submit mediated DNS datagram: {error}")))?;

‎crates/openshell-driver-docker/src/lib.rs‎

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,12 @@ const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30);
8383
const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal";
8484
const HOST_DOCKER_INTERNAL: &str = "host.docker.internal";
8585

86+
const HOST_TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload";
87+
/// The backend this driver's deployment admits, delivered to the supervisor on
88+
/// a channel separate from the topology descriptor so descriptor verification
89+
/// is not self-referential.
90+
const DRIVER_ADMITTED_BACKEND: &str = "docker";
91+
8692
fn provisioning_span(
8793
parent: &opentelemetry::Context,
8894
sandbox: &DriverSandbox,
@@ -526,6 +532,19 @@ impl DockerComputeDriver {
526532
Ok(driver)
527533
}
528534

535+
#[cfg(not(target_os = "linux"))]
536+
fn spawn_host_supervisor(
537+
&self,
538+
_sandbox: &DriverSandbox,
539+
_image: &DockerImageMetadata,
540+
_workspace_root: &str,
541+
) -> Result<Child, Status> {
542+
Err(Status::failed_precondition(
543+
"the Docker host-supervisor isolation backend is only supported on Linux",
544+
))
545+
}
546+
547+
#[cfg(target_os = "linux")]
529548
fn spawn_host_supervisor(
530549
&self,
531550
sandbox: &DriverSandbox,
@@ -607,6 +626,22 @@ impl DockerComputeDriver {
607626
))
608627
})?;
609628
}
629+
// The payload carries the boundary listener token, so it must not
630+
// appear in the world-readable process cmdline; deliver it through a
631+
// driver-owned 0600 file like the gateway token.
632+
let payload_path = state_dir.join(HOST_TOPOLOGY_PAYLOAD_FILE);
633+
std::fs::write(&payload_path, &descriptor.payload)
634+
.map_err(|error| Status::internal(format!("write Docker topology payload: {error}")))?;
635+
#[cfg(unix)]
636+
std::fs::set_permissions(&payload_path, {
637+
use std::os::unix::fs::PermissionsExt as _;
638+
std::fs::Permissions::from_mode(0o600)
639+
})
640+
.map_err(|error| Status::internal(format!("restrict Docker topology payload: {error}")))?;
641+
let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(
642+
sandbox.spec.as_ref(),
643+
)
644+
.map_err(|error| Status::internal(format!("encode main process spec: {error}")))?;
610645
let mut command = Command::new(supervisor_binary);
611646
command
612647
.kill_on_drop(true)
@@ -623,14 +658,18 @@ impl DockerComputeDriver {
623658
))
624659
.arg("--topology-backend-name=docker")
625660
.arg(format!("--topology-version={}", descriptor.version))
626-
.arg(format!(
627-
"--topology-payload-base64={}",
628-
base64::engine::general_purpose::STANDARD.encode(descriptor.payload)
629-
))
661+
.arg("--topology-payload-file")
662+
.arg(&payload_path)
630663
.arg("--workdir")
631664
.arg("/")
632-
.arg("--")
633-
.args(["/bin/sh", "-lc", "while :; do sleep 3600; done"])
665+
.env(
666+
openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND,
667+
DRIVER_ADMITTED_BACKEND,
668+
)
669+
.env(
670+
openshell_core::sandbox_env::MAIN_PROCESS_SPEC,
671+
main_process_spec,
672+
)
634673
.env(
635674
openshell_core::sandbox_env::ENDPOINT,
636675
&self.config.grpc_endpoint,

‎mise.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,10 @@ _.file = [".env"]
6161
KUBECONFIG = "{{config_root}}/kubeconfig"
6262
UV_CACHE_DIR = "{{config_root}}/.cache/uv"
6363

64-
# Enable sccache for faster Rust builds. Use one user-level OpenShell cache
65-
# shared by every worktree while preserving an override from the caller.
64+
# Enable sccache for faster Rust builds. Preserve an SCCACHE_DIR selected by the
65+
# caller so workstations and CI can configure their cache locations separately.
6666
RUSTC_WRAPPER = "sccache"
67-
SCCACHE_DIR = "{{ get_env(name='SCCACHE_DIR', default=xdg_cache_home ~ '/openshell/sccache') }}"
67+
SCCACHE_DIR = "{{ get_env(name='SCCACHE_DIR', default=config_root ~ '/.cache/sccache') }}"
6868

6969
# Shared build constants (overridable via environment).
7070
# DOCKER_BUILDKIT enables BuildKit for Docker; ignored by podman.

‎rfc/0012-isolation-backend/README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,12 +169,22 @@ struct SandboxContext {
169169
trait BoundBoundary: Send {
170170
fn network_mediation_source(&self) -> Arc<dyn NetworkMediationSource>;
171171
fn dns_mediation_source(&self) -> Option<Arc<dyn DnsMediationSource>>;
172+
fn host_gateway_ip(&self) -> Option<IpAddr>;
172173

173174
async fn confirm(
174175
self: Box<Self>,
175176
) -> Result<Box<dyn ReadyBoundary>, BackendError>;
176177
}
178+
```
179+
180+
`host_gateway_ip` is the backend's trusted host-side dial target for the
181+
well-known host-gateway aliases. A backend returns it when the mediation
182+
service runs outside the workload boundary and therefore cannot use the
183+
boundary's resolver view; the supervisor preserves the original hostname for
184+
policy, HTTP, and TLS while dialing the backend-provided address. `None`
185+
leaves host-gateway discovery to the supervisor's local environment.
177186

187+
```rust
178188
#[async_trait]
179189
trait ReadyBoundary: Send {
180190
async fn start_agent(
@@ -335,6 +345,8 @@ enum BackendErrorKind { Invalid, Denied, Unavailable, Failed, Terminated }
335345

336346
`Invalid` covers descriptor, version, and backend mismatches; `Denied` covers authenticated attachment rejection; `Unavailable` covers transient inability to serve an operation; `Failed` covers other backend faults; and `Terminated` reports boundary or workload termination, or an operation against an inactive boundary. An error never advances the lifecycle or authorizes an operation, and backend selection never falls back.
337347

348+
An `Unsupported` error variant reports that the selected backend does not implement an optional contract operation; it maps to the `Unavailable` kind for status purposes and never weakens a mandatory conformance requirement.
349+
338350
A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per orchestration attempt. Attach-time establishment uses sandbox identity and launch generation as an idempotency key. If the operation does not return `Bound`, the compute driver reclaims the topology rather than reusing an ambiguous resource.
339351

340352
Failures resolve as follows:

0 commit comments

Comments
 (0)