Skip to content

Commit aa74c33

Browse files
fix(supervisor): report rejected OCI working directories as WorkspaceValidationFailed
The RFC 0012 split dropped the supervisor exit status that drivers map to the WorkspaceValidationFailed condition. An image WORKDIR the sandbox identity cannot use then surfaced as ControlSupervisorStartFailed on Docker and as a signal kill on Podman. The supervisor now exits with the reserved status when the sandbox rejects the image working directory. Docker maps that supervisor exit during readiness and monitoring, and Podman maps the supervisor companion's exit instead of the workload's. Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
1 parent 2fed032 commit aa74c33

7 files changed

Lines changed: 170 additions & 21 deletions

File tree

‎crates/openshell-core/src/driver_utils.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,22 @@ pub const CONDITION_WORKSPACE_VALIDATION_FAILED: &str = "WorkspaceValidationFail
5858

5959
/// Supervisor exit status reserved for OCI workspace validation failures.
6060
///
61-
/// Local container drivers translate this status into
61+
/// Local container drivers translate a supervisor exit with this status into
6262
/// [`CONDITION_WORKSPACE_VALIDATION_FAILED`] so users receive the specific
6363
/// provisioning failure rather than a generic container exit.
6464
pub const SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED: i32 = 78;
6565

66+
/// Error context for a rejected image-provided OCI working directory.
67+
///
68+
/// The supervisor recognizes it in a failed agent start and exits with
69+
/// [`SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED`].
70+
pub const WORKSPACE_VALIDATION_ERROR_CONTEXT: &str = "image workspace validation failed";
71+
72+
/// Driver condition message for [`CONDITION_WORKSPACE_VALIDATION_FAILED`].
73+
/// Fixed text, because supervisor output may contain secrets.
74+
pub const WORKSPACE_VALIDATION_FAILED_MESSAGE: &str =
75+
"OCI WorkingDir is not usable by the sandbox identity";
76+
6677
/// Ready-condition reason when a container was terminated by an external signal.
6778
///
6879
/// SIGKILL/SIGTERM (exit 137/143) is what a Podman/Docker machine or daemon

‎crates/openshell-driver-docker/src/lib.rs‎

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,12 @@ use futures::{Stream, StreamExt};
2626
use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
2727
use openshell_core::driver_mounts;
2828
use openshell_core::driver_utils::{
29-
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE,
30-
LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE,
31-
SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh,
32-
temp_extract_container_name, validate_linux_elf_binary,
29+
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_WORKSPACE_VALIDATION_FAILED,
30+
LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME,
31+
LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SANDBOX_RUNTIME_IMAGE_BINARY_PATH,
32+
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_FAILED_MESSAGE,
33+
extract_first_tar_entry, supervisor_image_should_refresh, temp_extract_container_name,
34+
validate_linux_elf_binary,
3335
};
3436
use openshell_core::gpu::{
3537
CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements,
@@ -1854,7 +1856,7 @@ impl DockerComputeDriver {
18541856
.await;
18551857
cleanup_docker_boundary_state(sandbox, &self.config);
18561858
return Err(DockerProvisioningFailure::new(
1857-
"ControlSupervisorStartFailed",
1859+
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
18581860
status.message(),
18591861
));
18601862
}
@@ -2146,7 +2148,7 @@ impl DockerComputeDriver {
21462148
Err(status) => {
21472149
handle_docker_runtime_failure(
21482150
failure_context,
2149-
"ControlSupervisorExited",
2151+
supervisor_start_failure_reason(&status, "ControlSupervisorExited"),
21502152
format!(
21512153
"failed to start Docker control supervisor: {}",
21522154
status.message()
@@ -5286,9 +5288,11 @@ async fn spawn_docker_control_process(
52865288
).await;
52875289
return;
52885290
}
5291+
let mut reason = "ControlSupervisorExited";
52895292
let mut message = match result {
52905293
Some(Ok(status)) => {
52915294
warn!(%sandbox_id, status = status.status_code, "Docker supervisor container exited unexpectedly");
5295+
reason = supervisor_exit_reason(status.status_code);
52925296
format!("Docker supervisor container exited with status {}", status.status_code)
52935297
}
52945298
Some(Err(error)) => {
@@ -5319,12 +5323,7 @@ async fn spawn_docker_control_process(
53195323
if monitored_shutdown.load(Ordering::Acquire) {
53205324
return;
53215325
}
5322-
handle_docker_runtime_failure(
5323-
failure_context,
5324-
"ControlSupervisorExited",
5325-
message,
5326-
)
5327-
.await;
5326+
handle_docker_runtime_failure(failure_context, reason, message).await;
53285327
},
53295328
}
53305329
});
@@ -5373,6 +5372,13 @@ async fn wait_for_docker_supervisor_ready(
53735372
let state = inspected.state.unwrap_or_default();
53745373
match state.health.and_then(|health| health.status) {
53755374
Some(HealthStatusEnum::HEALTHY) => return Ok(()),
5375+
_ if state.running == Some(false)
5376+
&& state.exit_code
5377+
== Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)) =>
5378+
{
5379+
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
5380+
return Err(workspace_validation_status(&log_tail));
5381+
}
53765382
_ if state.running == Some(false) => {
53775383
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
53785384
let sandbox_log_tail = docker_container_log_tail(docker, sandbox_id).await;
@@ -5387,6 +5393,38 @@ async fn wait_for_docker_supervisor_ready(
53875393
}
53885394
}
53895395

5396+
/// Startup failure for a supervisor that exited because the sandbox rejected
5397+
/// the image working directory. The fixed message prefix identifies it for
5398+
/// [`supervisor_start_failure_reason`].
5399+
fn workspace_validation_status(log_tail: &str) -> Status {
5400+
Status::failed_precondition(format!(
5401+
"{WORKSPACE_VALIDATION_FAILED_MESSAGE}{}",
5402+
format_log_tail(log_tail)
5403+
))
5404+
}
5405+
5406+
/// Condition reason for a supervisor that failed before becoming ready.
5407+
fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &'static str {
5408+
if status.code() == tonic::Code::FailedPrecondition
5409+
&& status
5410+
.message()
5411+
.starts_with(WORKSPACE_VALIDATION_FAILED_MESSAGE)
5412+
{
5413+
CONDITION_WORKSPACE_VALIDATION_FAILED
5414+
} else {
5415+
default
5416+
}
5417+
}
5418+
5419+
/// Condition reason for a supervisor that exited with `status_code`.
5420+
fn supervisor_exit_reason(status_code: i64) -> &'static str {
5421+
if status_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) {
5422+
CONDITION_WORKSPACE_VALIDATION_FAILED
5423+
} else {
5424+
"ControlSupervisorExited"
5425+
}
5426+
}
5427+
53905428
fn format_log_tail(log_tail: &str) -> String {
53915429
format_named_log_tail("log tail", log_tail)
53925430
}

‎crates/openshell-driver-docker/src/tests.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3595,6 +3595,28 @@ fn docker_oom_kill_stays_terminal_despite_137() {
35953595
assert_eq!(ready_reason(&sandbox), CONDITION_EXITED);
35963596
}
35973597

3598+
#[test]
3599+
fn supervisor_workspace_validation_exit_is_reported_explicitly() {
3600+
let status = workspace_validation_status("image workspace validation failed: denied");
3601+
assert!(status.message().contains("log tail: image workspace"));
3602+
assert_eq!(
3603+
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
3604+
CONDITION_WORKSPACE_VALIDATION_FAILED
3605+
);
3606+
assert_eq!(
3607+
supervisor_start_failure_reason(
3608+
&Status::failed_precondition("provider SPIFFE socket has no parent directory"),
3609+
"ControlSupervisorStartFailed"
3610+
),
3611+
"ControlSupervisorStartFailed"
3612+
);
3613+
assert_eq!(
3614+
supervisor_exit_reason(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)),
3615+
CONDITION_WORKSPACE_VALIDATION_FAILED
3616+
);
3617+
assert_eq!(supervisor_exit_reason(1), "ControlSupervisorExited");
3618+
}
3619+
35983620
#[test]
35993621
fn concurrent_container_removal_is_idempotent() {
36003622
let removing = BollardError::DockerResponseServerError {

‎crates/openshell-driver-podman/src/client.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,10 @@ pub struct ContainerState {
121121
/// container-log marker. It is never deserialized from Podman.
122122
#[serde(skip)]
123123
pub startup_diagnostic: Option<String>,
124+
/// Exit status of the sandbox's supervisor companion when it stopped
125+
/// before the workload. It is never deserialized from Podman.
126+
#[serde(skip)]
127+
pub supervisor_exit_code: Option<i64>,
124128
}
125129

126130
#[derive(Debug, Clone, serde::Deserialize)]

‎crates/openshell-driver-podman/src/watcher.rs‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ const CONDITION_STARTING: &str = "ContainerStarting";
2929
use openshell_core::driver_utils::{
3030
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_STOPPED,
3131
CONDITION_WORKSPACE_VALIDATION_FAILED, SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED,
32+
WORKSPACE_VALIDATION_FAILED_MESSAGE,
3233
};
3334

3435
pub type WatchStream =
@@ -390,7 +391,12 @@ pub async fn inspect_workload(
390391
// Both containers exist before initial start. A missing or exited
391392
// companion therefore requires containment, including after a
392393
// gateway restart that missed the original Podman exit event.
393-
Ok(_) | Err(PodmanApiError::NotFound(_)) => {
394+
Ok(supervisor) => {
395+
client.stop_container(&workload.id, 0).await?;
396+
workload = client.inspect_container(&workload.id).await?;
397+
workload.state.supervisor_exit_code = exited_exit_code(&supervisor.state);
398+
}
399+
Err(PodmanApiError::NotFound(_)) => {
394400
client.stop_container(&workload.id, 0).await?;
395401
workload = client.inspect_container(&workload.id).await?;
396402
}
@@ -402,10 +408,17 @@ pub async fn inspect_workload(
402408
.await
403409
.ok()
404410
.and_then(|logs| boundary_startup_termination_marker(&logs));
411+
workload.state.supervisor_exit_code = supervisor
412+
.ok()
413+
.and_then(|supervisor| exited_exit_code(&supervisor.state));
405414
}
406415
Ok(workload)
407416
}
408417

418+
fn exited_exit_code(state: &ContainerState) -> Option<i64> {
419+
matches!(state.status.as_str(), "exited" | "stopped").then_some(state.exit_code)
420+
}
421+
409422
/// Extract only fixed, OpenShell-owned startup diagnostics from container
410423
/// output. Workload and supervisor output may contain secrets, so it must not
411424
/// be propagated to driver conditions or tracing.
@@ -553,10 +566,12 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition {
553566
"OOMKilled",
554567
"Container was killed by the OOM killer".to_string(),
555568
)
556-
} else if state.exit_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) {
569+
} else if state.supervisor_exit_code
570+
== Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED))
571+
{
557572
(
558573
CONDITION_WORKSPACE_VALIDATION_FAILED,
559-
"OCI WorkingDir is not usable by the sandbox identity".to_string(),
574+
WORKSPACE_VALIDATION_FAILED_MESSAGE.to_string(),
560575
)
561576
} else if matches!(state.exit_code, 137 | 143) {
562577
(
@@ -692,6 +707,7 @@ mod tests {
692707
health: None,
693708
started_at: Some("2026-08-12T16:38:58Z".to_string()),
694709
finished_at: Some("2026-08-12T16:39:13Z".to_string()),
710+
supervisor_exit_code: None,
695711
startup_diagnostic: None,
696712
};
697713

@@ -741,6 +757,7 @@ mod tests {
741757
}),
742758
started_at: Some("2026-04-14T10:00:00Z".to_string()),
743759
finished_at: None,
760+
supervisor_exit_code: None,
744761
startup_diagnostic: None,
745762
};
746763
let cond = condition_from_state(&state);
@@ -760,6 +777,7 @@ mod tests {
760777
health: None,
761778
started_at: Some("2026-04-14T10:00:00Z".to_string()),
762779
finished_at: None,
780+
supervisor_exit_code: None,
763781
startup_diagnostic: None,
764782
};
765783
let cond = condition_from_state(&state);
@@ -782,6 +800,7 @@ mod tests {
782800
}),
783801
started_at: Some("2026-04-14T10:00:00Z".to_string()),
784802
finished_at: None,
803+
supervisor_exit_code: None,
785804
startup_diagnostic: None,
786805
};
787806
let condition = condition_from_state(&state);
@@ -800,6 +819,7 @@ mod tests {
800819
health: None,
801820
started_at: None,
802821
finished_at: Some("2026-04-14T11:00:00Z".to_string()),
822+
supervisor_exit_code: None,
803823
startup_diagnostic: None,
804824
};
805825
let cond = condition_from_state(&state);
@@ -818,6 +838,7 @@ mod tests {
818838
health: None,
819839
started_at: None,
820840
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
841+
supervisor_exit_code: None,
821842
startup_diagnostic: None,
822843
};
823844
let cond = condition_from_state(&state);
@@ -836,6 +857,7 @@ mod tests {
836857
health: None,
837858
started_at: None,
838859
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
860+
supervisor_exit_code: None,
839861
startup_diagnostic: boundary_startup_termination_marker(
840862
b"untrusted workload output\nsandbox boundary received SIGTERM before supervisor confirmation\n",
841863
),
@@ -860,21 +882,30 @@ mod tests {
860882

861883
#[test]
862884
fn condition_workspace_validation_exit_is_reported_explicitly() {
863-
let state = ContainerState {
885+
// The supervisor exits with the reserved status and the watcher then
886+
// stops the workload, so the workload itself reports SIGKILL.
887+
let mut state = ContainerState {
864888
status: "exited".to_string(),
865889
running: false,
866-
exit_code: i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED),
890+
exit_code: 137,
867891
oom_killed: false,
868892
health: None,
869893
started_at: None,
870894
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
895+
supervisor_exit_code: Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)),
871896
startup_diagnostic: None,
872897
};
873898

874899
let cond = condition_from_state(&state);
875900

876901
assert_eq!(cond.reason, CONDITION_WORKSPACE_VALIDATION_FAILED);
877-
assert!(cond.message.contains("WorkingDir"));
902+
assert_eq!(cond.message, WORKSPACE_VALIDATION_FAILED_MESSAGE);
903+
904+
// A workload's own exit with the reserved status is not a supervisor
905+
// workspace rejection.
906+
state.exit_code = i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED);
907+
state.supervisor_exit_code = Some(1);
908+
assert_eq!(condition_from_state(&state).reason, CONDITION_EXITED);
878909
}
879910

880911
#[test]
@@ -892,6 +923,7 @@ mod tests {
892923
health: None,
893924
started_at: None,
894925
finished_at: Some("2026-04-14T12:30:00Z".to_string()),
926+
supervisor_exit_code: None,
895927
startup_diagnostic: None,
896928
};
897929
let cond = condition_from_state(&state);

‎crates/openshell-sandbox/src/delegated.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ pub async fn spawn_workload(
5454
crate::process::validate_oci_workspace_as_effective_identity(std::path::Path::new(
5555
workspace_root,
5656
))
57-
.wrap_err("image workspace validation failed")?;
57+
.wrap_err(openshell_core::driver_utils::WORKSPACE_VALIDATION_ERROR_CONTEXT)?;
5858
}
5959

6060
#[cfg(target_os = "linux")]

0 commit comments

Comments
 (0)