Skip to content

Commit 7b9ebeb

Browse files
committed
fix(docker): preserve provisioning failure status
Signed-off-by: Evan Lezar <elezar@nvidia.com>
1 parent 26f2f96 commit 7b9ebeb

2 files changed

Lines changed: 96 additions & 5 deletions

File tree

‎crates/openshell-driver-docker/src/lib.rs‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,14 @@ impl DockerComputeDriver {
847847
sandbox_id: &str,
848848
sandbox_name: &str,
849849
) -> Result<Option<DriverSandbox>, Status> {
850+
let pending = self.pending_snapshot(sandbox_id, sandbox_name).await?;
851+
if pending
852+
.as_ref()
853+
.is_some_and(pending_sandbox_has_provisioning_failure)
854+
{
855+
return Ok(pending);
856+
}
857+
850858
let container = self
851859
.find_managed_container_summary(sandbox_id, sandbox_name)
852860
.await?;
@@ -856,7 +864,7 @@ impl DockerComputeDriver {
856864
return Ok(Some(sandbox));
857865
}
858866

859-
self.pending_snapshot(sandbox_id, sandbox_name).await
867+
Ok(pending)
860868
}
861869

862870
async fn current_snapshots(&self) -> Result<Vec<DriverSandbox>, Status> {
@@ -892,10 +900,11 @@ impl DockerComputeDriver {
892900
}
893901
container_sandboxes.push(sandbox);
894902
}
895-
let mut by_id = self.pending_snapshot_map().await;
896-
for sandbox in container_sandboxes {
897-
by_id.insert(sandbox.id.clone(), sandbox);
898-
}
903+
let mut by_id = container_sandboxes
904+
.into_iter()
905+
.map(|sandbox| (sandbox.id.clone(), sandbox))
906+
.collect::<HashMap<_, _>>();
907+
merge_pending_sandbox_snapshots(&mut by_id, self.pending_snapshot_map().await);
899908
let mut sandboxes = by_id.into_values().collect::<Vec<_>>();
900909
sandboxes.sort_by(|left, right| left.id.cmp(&right.id));
901910
Ok(sandboxes)
@@ -2296,6 +2305,33 @@ fn resolve_pending_id(
22962305
Ok(Some(id))
22972306
}
22982307

2308+
/// A pending sandbox holds either the ordinary in-progress snapshot or the
2309+
/// explicit error published by its provisioning task. The latter is more
2310+
/// informative than a transient Docker state observed while that task cleans
2311+
/// up a failed start, so it must win during snapshot reconciliation.
2312+
fn pending_sandbox_has_provisioning_failure(sandbox: &DriverSandbox) -> bool {
2313+
sandbox.status.as_ref().is_some_and(|status| {
2314+
status.conditions.iter().any(|condition| {
2315+
condition.r#type == "Ready"
2316+
&& condition.status.eq_ignore_ascii_case("false")
2317+
&& condition.reason != "Starting"
2318+
})
2319+
})
2320+
}
2321+
2322+
fn merge_pending_sandbox_snapshots(
2323+
snapshots: &mut HashMap<String, DriverSandbox>,
2324+
pending: HashMap<String, DriverSandbox>,
2325+
) {
2326+
for (sandbox_id, sandbox) in pending {
2327+
if pending_sandbox_has_provisioning_failure(&sandbox) {
2328+
snapshots.insert(sandbox_id, sandbox);
2329+
} else {
2330+
snapshots.entry(sandbox_id).or_insert(sandbox);
2331+
}
2332+
}
2333+
}
2334+
22992335
fn provisioning_condition() -> DriverCondition {
23002336
DriverCondition {
23012337
r#type: "Ready".to_string(),

‎crates/openshell-driver-docker/src/tests.rs‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3129,6 +3129,61 @@ fn pending_sandbox_snapshot_uses_docker_namespace_and_starting_condition() {
31293129
assert_eq!(status.conditions[0].message, "Docker container is starting");
31303130
}
31313131

3132+
#[test]
3133+
fn pending_provisioning_failure_overrides_transient_container_state() {
3134+
let sandbox = test_sandbox();
3135+
let dead_container = pending_sandbox_snapshot(
3136+
&sandbox,
3137+
"default",
3138+
error_condition("ContainerDead", "Container is dead"),
3139+
false,
3140+
);
3141+
let start_failure = pending_sandbox_snapshot(
3142+
&sandbox,
3143+
"default",
3144+
error_condition(
3145+
"ContainerStartFailed",
3146+
"Docker responded with status code 500: CDI device injection failed",
3147+
),
3148+
false,
3149+
);
3150+
3151+
let mut snapshots = HashMap::from([(sandbox.id.clone(), dead_container)]);
3152+
merge_pending_sandbox_snapshots(
3153+
&mut snapshots,
3154+
HashMap::from([(sandbox.id.clone(), start_failure)]),
3155+
);
3156+
3157+
let status = snapshots[&sandbox.id].status.as_ref().expect("status");
3158+
assert_eq!(status.conditions[0].reason, "ContainerStartFailed");
3159+
assert!(
3160+
status.conditions[0]
3161+
.message
3162+
.contains("CDI device injection failed")
3163+
);
3164+
}
3165+
3166+
#[test]
3167+
fn pending_starting_snapshot_does_not_override_container_state() {
3168+
let sandbox = test_sandbox();
3169+
let dead_container = pending_sandbox_snapshot(
3170+
&sandbox,
3171+
"default",
3172+
error_condition("ContainerDead", "Container is dead"),
3173+
false,
3174+
);
3175+
let starting = pending_sandbox_snapshot(&sandbox, "default", provisioning_condition(), false);
3176+
3177+
let mut snapshots = HashMap::from([(sandbox.id.clone(), dead_container)]);
3178+
merge_pending_sandbox_snapshots(
3179+
&mut snapshots,
3180+
HashMap::from([(sandbox.id.clone(), starting)]),
3181+
);
3182+
3183+
let status = snapshots[&sandbox.id].status.as_ref().expect("status");
3184+
assert_eq!(status.conditions[0].reason, "ContainerDead");
3185+
}
3186+
31323187
#[test]
31333188
fn validate_linux_elf_binary_rejects_non_elf_files() {
31343189
let tempdir = TempDir::new().unwrap();

0 commit comments

Comments
 (0)