Skip to content

Commit 93cd614

Browse files
fix(supervisor): report rejected OCI working directories as WorkspaceValidationFailed
The RFC 0012 split dropped the supervisor exit status that drivers map to the WorkspaceValidationFailed condition. An image WORKDIR the sandbox identity cannot use then surfaced as ControlSupervisorStartFailed on Docker and as a signal kill on Podman. The supervisor now exits with the reserved status when the sandbox rejects the image working directory. Docker maps that supervisor exit during readiness and monitoring, and Podman maps the supervisor companion's exit instead of the workload's. Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
1 parent 53e7b71 commit 93cd614

7 files changed

Lines changed: 170 additions & 21 deletions

File tree

‎crates/openshell-core/src/driver_utils.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,22 @@ pub const CONDITION_WORKSPACE_VALIDATION_FAILED: &str = "WorkspaceValidationFail
5858

5959
/// Supervisor exit status reserved for OCI workspace validation failures.
6060
///
61-
/// Local container drivers translate this status into
61+
/// Local container drivers translate a supervisor exit with this status into
6262
/// [`CONDITION_WORKSPACE_VALIDATION_FAILED`] so users receive the specific
6363
/// provisioning failure rather than a generic container exit.
6464
pub const SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED: i32 = 78;
6565

66+
/// Error context for a rejected image-provided OCI working directory.
67+
///
68+
/// The supervisor recognizes it in a failed agent start and exits with
69+
/// [`SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED`].
70+
pub const WORKSPACE_VALIDATION_ERROR_CONTEXT: &str = "image workspace validation failed";
71+
72+
/// Driver condition message for [`CONDITION_WORKSPACE_VALIDATION_FAILED`].
73+
/// Fixed text, because supervisor output may contain secrets.
74+
pub const WORKSPACE_VALIDATION_FAILED_MESSAGE: &str =
75+
"OCI WorkingDir is not usable by the sandbox identity";
76+
6677
/// Ready-condition reason when a container was terminated by an external signal.
6778
///
6879
/// SIGKILL/SIGTERM (exit 137/143) is what a Podman/Docker machine or daemon

‎crates/openshell-driver-docker/src/lib.rs‎

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,12 @@ use futures::{Stream, StreamExt};
2626
use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
2727
use openshell_core::driver_mounts;
2828
use openshell_core::driver_utils::{
29-
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE,
30-
LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE,
31-
SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh,
32-
temp_extract_container_name, validate_linux_elf_binary,
29+
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_WORKSPACE_VALIDATION_FAILED,
30+
LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME,
31+
LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SANDBOX_RUNTIME_IMAGE_BINARY_PATH,
32+
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_FAILED_MESSAGE,
33+
extract_first_tar_entry, supervisor_image_should_refresh, temp_extract_container_name,
34+
validate_linux_elf_binary,
3335
};
3436
use openshell_core::gpu::{
3537
CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements,
@@ -1854,7 +1856,7 @@ impl DockerComputeDriver {
18541856
.await;
18551857
cleanup_docker_boundary_state(sandbox, &self.config);
18561858
return Err(DockerProvisioningFailure::new(
1857-
"ControlSupervisorStartFailed",
1859+
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
18581860
status.message(),
18591861
));
18601862
}
@@ -2146,7 +2148,7 @@ impl DockerComputeDriver {
21462148
Err(status) => {
21472149
handle_docker_runtime_failure(
21482150
failure_context,
2149-
"ControlSupervisorExited",
2151+
supervisor_start_failure_reason(&status, "ControlSupervisorExited"),
21502152
format!(
21512153
"failed to start Docker control supervisor: {}",
21522154
status.message()
@@ -5272,9 +5274,11 @@ async fn spawn_docker_control_process(
52725274
).await;
52735275
return;
52745276
}
5277+
let mut reason = "ControlSupervisorExited";
52755278
let mut message = match result {
52765279
Some(Ok(status)) => {
52775280
warn!(%sandbox_id, status = status.status_code, "Docker supervisor container exited unexpectedly");
5281+
reason = supervisor_exit_reason(status.status_code);
52785282
format!("Docker supervisor container exited with status {}", status.status_code)
52795283
}
52805284
Some(Err(error)) => {
@@ -5299,12 +5303,7 @@ async fn spawn_docker_control_process(
52995303
if monitored_shutdown.load(Ordering::Acquire) {
53005304
return;
53015305
}
5302-
handle_docker_runtime_failure(
5303-
failure_context,
5304-
"ControlSupervisorExited",
5305-
message,
5306-
)
5307-
.await;
5306+
handle_docker_runtime_failure(failure_context, reason, message).await;
53085307
},
53095308
}
53105309
});
@@ -5351,6 +5350,13 @@ async fn wait_for_docker_supervisor_ready(
53515350
let state = inspected.state.unwrap_or_default();
53525351
match state.health.and_then(|health| health.status) {
53535352
Some(HealthStatusEnum::HEALTHY) => return Ok(()),
5353+
_ if state.running == Some(false)
5354+
&& state.exit_code
5355+
== Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)) =>
5356+
{
5357+
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
5358+
return Err(workspace_validation_status(&log_tail));
5359+
}
53545360
_ if state.running == Some(false) => {
53555361
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
53565362
warn!(sandbox_id, supervisor_id, supervisor_logs = %log_tail, "Docker supervisor exited before becoming ready");
@@ -5364,6 +5370,38 @@ async fn wait_for_docker_supervisor_ready(
53645370
}
53655371
}
53665372

5373+
/// Startup failure for a supervisor that exited because the sandbox rejected
5374+
/// the image working directory. The fixed message prefix identifies it for
5375+
/// [`supervisor_start_failure_reason`].
5376+
fn workspace_validation_status(log_tail: &str) -> Status {
5377+
Status::failed_precondition(format!(
5378+
"{WORKSPACE_VALIDATION_FAILED_MESSAGE}{}",
5379+
format_log_tail(log_tail)
5380+
))
5381+
}
5382+
5383+
/// Condition reason for a supervisor that failed before becoming ready.
5384+
fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &'static str {
5385+
if status.code() == tonic::Code::FailedPrecondition
5386+
&& status
5387+
.message()
5388+
.starts_with(WORKSPACE_VALIDATION_FAILED_MESSAGE)
5389+
{
5390+
CONDITION_WORKSPACE_VALIDATION_FAILED
5391+
} else {
5392+
default
5393+
}
5394+
}
5395+
5396+
/// Condition reason for a supervisor that exited with `status_code`.
5397+
fn supervisor_exit_reason(status_code: i64) -> &'static str {
5398+
if status_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) {
5399+
CONDITION_WORKSPACE_VALIDATION_FAILED
5400+
} else {
5401+
"ControlSupervisorExited"
5402+
}
5403+
}
5404+
53675405
fn format_log_tail(log_tail: &str) -> String {
53685406
// gRPC status messages travel in HTTP/2 headers. A full 16 KiB container
53695407
// tail can exceed the client's 16 KiB header budget and hide the real

‎crates/openshell-driver-docker/src/tests.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3619,6 +3619,28 @@ fn docker_oom_kill_stays_terminal_despite_137() {
36193619
assert_eq!(ready_reason(&sandbox), CONDITION_EXITED);
36203620
}
36213621

3622+
#[test]
3623+
fn supervisor_workspace_validation_exit_is_reported_explicitly() {
3624+
let status = workspace_validation_status("image workspace validation failed: denied");
3625+
assert!(status.message().contains("log tail: image workspace"));
3626+
assert_eq!(
3627+
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
3628+
CONDITION_WORKSPACE_VALIDATION_FAILED
3629+
);
3630+
assert_eq!(
3631+
supervisor_start_failure_reason(
3632+
&Status::failed_precondition("provider SPIFFE socket has no parent directory"),
3633+
"ControlSupervisorStartFailed"
3634+
),
3635+
"ControlSupervisorStartFailed"
3636+
);
3637+
assert_eq!(
3638+
supervisor_exit_reason(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)),
3639+
CONDITION_WORKSPACE_VALIDATION_FAILED
3640+
);
3641+
assert_eq!(supervisor_exit_reason(1), "ControlSupervisorExited");
3642+
}
3643+
36223644
#[test]
36233645
fn concurrent_container_removal_is_idempotent() {
36243646
let removing = BollardError::DockerResponseServerError {

‎crates/openshell-driver-podman/src/client.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,10 @@ pub struct ContainerState {
121121
/// container-log marker. It is never deserialized from Podman.
122122
#[serde(skip)]
123123
pub startup_diagnostic: Option<String>,
124+
/// Exit status of the sandbox's supervisor companion when it stopped
125+
/// before the workload. It is never deserialized from Podman.
126+
#[serde(skip)]
127+
pub supervisor_exit_code: Option<i64>,
124128
}
125129

126130
#[derive(Debug, Clone, serde::Deserialize)]

‎crates/openshell-driver-podman/src/watcher.rs‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ const CONDITION_STARTING: &str = "ContainerStarting";
2929
use openshell_core::driver_utils::{
3030
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_STOPPED,
3131
CONDITION_WORKSPACE_VALIDATION_FAILED, SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED,
32+
WORKSPACE_VALIDATION_FAILED_MESSAGE,
3233
};
3334

3435
pub type WatchStream =
@@ -390,7 +391,12 @@ pub async fn inspect_workload(
390391
// Both containers exist before initial start. A missing or exited
391392
// companion therefore requires containment, including after a
392393
// gateway restart that missed the original Podman exit event.
393-
Ok(_) | Err(PodmanApiError::NotFound(_)) => {
394+
Ok(supervisor) => {
395+
client.stop_container(&workload.id, 0).await?;
396+
workload = client.inspect_container(&workload.id).await?;
397+
workload.state.supervisor_exit_code = exited_exit_code(&supervisor.state);
398+
}
399+
Err(PodmanApiError::NotFound(_)) => {
394400
client.stop_container(&workload.id, 0).await?;
395401
workload = client.inspect_container(&workload.id).await?;
396402
}
@@ -402,10 +408,17 @@ pub async fn inspect_workload(
402408
.await
403409
.ok()
404410
.and_then(|logs| boundary_startup_termination_marker(&logs));
411+
workload.state.supervisor_exit_code = supervisor
412+
.ok()
413+
.and_then(|supervisor| exited_exit_code(&supervisor.state));
405414
}
406415
Ok(workload)
407416
}
408417

418+
fn exited_exit_code(state: &ContainerState) -> Option<i64> {
419+
matches!(state.status.as_str(), "exited" | "stopped").then_some(state.exit_code)
420+
}
421+
409422
/// Extract only fixed, OpenShell-owned startup diagnostics from container
410423
/// output. Workload and supervisor output may contain secrets, so it must not
411424
/// be propagated to driver conditions or tracing.
@@ -553,10 +566,12 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition {
553566
"OOMKilled",
554567
"Container was killed by the OOM killer".to_string(),
555568
)
556-
} else if state.exit_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) {
569+
} else if state.supervisor_exit_code
570+
== Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED))
571+
{
557572
(
558573
CONDITION_WORKSPACE_VALIDATION_FAILED,
559-
"OCI WorkingDir is not usable by the sandbox identity".to_string(),
574+
WORKSPACE_VALIDATION_FAILED_MESSAGE.to_string(),
560575
)
561576
} else if matches!(state.exit_code, 137 | 143) {
562577
(
@@ -692,6 +707,7 @@ mod tests {
692707
health: None,
693708
started_at: Some("2026-08-12T16:38:58Z".to_string()),
694709
finished_at: Some("2026-08-12T16:39:13Z".to_string()),
710+
supervisor_exit_code: None,
695711
startup_diagnostic: None,
696712
};
697713

@@ -741,6 +757,7 @@ mod tests {
741757
}),
742758
started_at: Some("2026-04-14T10:00:00Z".to_string()),
743759
finished_at: None,
760+
supervisor_exit_code: None,
744761
startup_diagnostic: None,
745762
};
746763
let cond = condition_from_state(&state);
@@ -760,6 +777,7 @@ mod tests {
760777
health: None,
761778
started_at: Some("2026-04-14T10:00:00Z".to_string()),
762779
finished_at: None,
780+
supervisor_exit_code: None,
763781
startup_diagnostic: None,
764782
};
765783
let cond = condition_from_state(&state);
@@ -782,6 +800,7 @@ mod tests {
782800
}),
783801
started_at: Some("2026-04-14T10:00:00Z".to_string()),
784802
finished_at: None,
803+
supervisor_exit_code: None,
785804
startup_diagnostic: None,
786805
};
787806
let condition = condition_from_state(&state);
@@ -800,6 +819,7 @@ mod tests {
800819
health: None,
801820
started_at: None,
802821
finished_at: Some("2026-04-14T11:00:00Z".to_string()),
822+
supervisor_exit_code: None,
803823
startup_diagnostic: None,
804824
};
805825
let cond = condition_from_state(&state);
@@ -818,6 +838,7 @@ mod tests {
818838
health: None,
819839
started_at: None,
820840
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
841+
supervisor_exit_code: None,
821842
startup_diagnostic: None,
822843
};
823844
let cond = condition_from_state(&state);
@@ -836,6 +857,7 @@ mod tests {
836857
health: None,
837858
started_at: None,
838859
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
860+
supervisor_exit_code: None,
839861
startup_diagnostic: boundary_startup_termination_marker(
840862
b"untrusted workload output\nsandbox boundary received SIGTERM before supervisor confirmation\n",
841863
),
@@ -860,21 +882,30 @@ mod tests {
860882

861883
#[test]
862884
fn condition_workspace_validation_exit_is_reported_explicitly() {
863-
let state = ContainerState {
885+
// The supervisor exits with the reserved status and the watcher then
886+
// stops the workload, so the workload itself reports SIGKILL.
887+
let mut state = ContainerState {
864888
status: "exited".to_string(),
865889
running: false,
866-
exit_code: i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED),
890+
exit_code: 137,
867891
oom_killed: false,
868892
health: None,
869893
started_at: None,
870894
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
895+
supervisor_exit_code: Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)),
871896
startup_diagnostic: None,
872897
};
873898

874899
let cond = condition_from_state(&state);
875900

876901
assert_eq!(cond.reason, CONDITION_WORKSPACE_VALIDATION_FAILED);
877-
assert!(cond.message.contains("WorkingDir"));
902+
assert_eq!(cond.message, WORKSPACE_VALIDATION_FAILED_MESSAGE);
903+
904+
// A workload's own exit with the reserved status is not a supervisor
905+
// workspace rejection.
906+
state.exit_code = i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED);
907+
state.supervisor_exit_code = Some(1);
908+
assert_eq!(condition_from_state(&state).reason, CONDITION_EXITED);
878909
}
879910

880911
#[test]
@@ -892,6 +923,7 @@ mod tests {
892923
health: None,
893924
started_at: None,
894925
finished_at: Some("2026-04-14T12:30:00Z".to_string()),
926+
supervisor_exit_code: None,
895927
startup_diagnostic: None,
896928
};
897929
let cond = condition_from_state(&state);

‎crates/openshell-sandbox/src/delegated.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ pub async fn spawn_workload(
5454
crate::process::validate_oci_workspace_as_effective_identity(std::path::Path::new(
5555
workspace_root,
5656
))
57-
.wrap_err("image workspace validation failed")?;
57+
.wrap_err(openshell_core::driver_utils::WORKSPACE_VALIDATION_ERROR_CONTEXT)?;
5858
}
5959

6060
#[cfg(target_os = "linux")]

0 commit comments

Comments
 (0)