Skip to content

Commit c9e8fba

Browse files
committed
feat(gpu): select counted CDI defaults
Signed-off-by: Evan Lezar <elezar@nvidia.com>
1 parent b7a3226 commit c9e8fba

9 files changed

Lines changed: 659 additions & 494 deletions

File tree

‎crates/openshell-core/src/gpu.rs‎

Lines changed: 294 additions & 126 deletions
Large diffs are not rendered by default.

‎crates/openshell-driver-docker/src/lib.rs‎

Lines changed: 150 additions & 133 deletions
Large diffs are not rendered by default.

‎crates/openshell-driver-docker/src/tests.rs‎

Lines changed: 42 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,6 @@ fn runtime_config() -> DockerDriverRuntimeConfig {
111111
}),
112112
daemon_version: "28.0.0".to_string(),
113113
supports_gpu: false,
114-
cdi_gpu_inventory: CdiGpuInventory::default(),
115114
allow_all_default_gpu: false,
116115
sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT,
117116
enable_bind_mounts: false,
@@ -179,6 +178,7 @@ impl SupervisorReadiness for DisconnectedSupervisorReadiness {
179178
}
180179

181180
fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDriver {
181+
let allow_all_default_gpu = config.allow_all_default_gpu;
182182
DockerComputeDriver {
183183
docker: Arc::new(
184184
Docker::connect_with_http("http://127.0.0.1:2375", 1, bollard::API_DEFAULT_VERSION)
@@ -188,7 +188,10 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr
188188
events: broadcast::channel(WATCH_BUFFER).0,
189189
pending: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
190190
supervisor_readiness: Arc::new(DisconnectedSupervisorReadiness),
191-
gpu_selector: Arc::new(CdiGpuRoundRobin::new()),
191+
gpu_selector: Arc::new(CdiGpuDefaultSelector::new(
192+
CdiGpuInventory::default(),
193+
allow_all_default_gpu,
194+
)),
192195
}
193196
}
194197

@@ -1105,19 +1108,14 @@ fn validate_sandbox_rejects_unknown_driver_config_fields() {
11051108
}
11061109

11071110
#[test]
1108-
fn validate_sandbox_rejects_gpu_count_request() {
1111+
fn validate_sandbox_accepts_gpu_count_request_shape() {
11091112
let mut config = runtime_config();
11101113
config.supports_gpu = true;
11111114
let mut sandbox = test_sandbox();
11121115
sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(Some(2)));
11131116

1114-
let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err();
1115-
1116-
assert_eq!(err.code(), tonic::Code::InvalidArgument);
1117-
assert!(
1118-
err.message()
1119-
.contains("GPU count requests are not supported")
1120-
);
1117+
DockerComputeDriver::validate_sandbox(&sandbox, &config)
1118+
.expect("default GPU count shape should be accepted before inventory selection");
11211119
}
11221120

11231121
#[test]
@@ -1166,7 +1164,7 @@ fn validate_sandbox_rejects_multiple_cdi_devices_without_gpu_count() {
11661164
assert_eq!(err.code(), tonic::Code::InvalidArgument);
11671165
assert!(
11681166
err.message()
1169-
.contains("requires an explicit gpu count matching its length (2)")
1167+
.contains("gpu count (1) must match driver_config.cdi_devices length (2)")
11701168
);
11711169
}
11721170

@@ -1253,9 +1251,15 @@ fn build_container_create_body_maps_default_gpu_to_selected_cdi_device() {
12531251
let mut sandbox = test_sandbox();
12541252
sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None));
12551253

1256-
let create_body =
1257-
build_container_create_body_with_default(&sandbox, &config, Some("nvidia.com/gpu=1"))
1258-
.unwrap();
1254+
let driver_config = DockerSandboxDriverConfig::default();
1255+
let gpu_devices = vec!["nvidia.com/gpu=1".to_string()];
1256+
let create_body = build_container_create_body_with_gpu_devices(
1257+
&sandbox,
1258+
&config,
1259+
&driver_config,
1260+
Some(&gpu_devices),
1261+
)
1262+
.unwrap();
12591263
let request = create_body
12601264
.host_config
12611265
.as_ref()
@@ -1271,19 +1275,20 @@ fn build_container_create_body_maps_default_gpu_to_selected_cdi_device() {
12711275
}
12721276

12731277
#[test]
1274-
fn build_container_create_body_rejects_missing_default_cdi_device() {
1278+
fn build_container_create_body_omits_devices_without_resolved_default_cdi_devices() {
12751279
let mut config = runtime_config();
12761280
config.supports_gpu = true;
12771281
let mut sandbox = test_sandbox();
12781282
sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None));
12791283

1280-
let err = build_container_create_body(&sandbox, &config).unwrap_err();
1284+
let create_body = build_container_create_body(&sandbox, &config).unwrap();
12811285

1282-
assert_eq!(err.code(), tonic::Code::FailedPrecondition);
12831286
assert!(
1284-
err.message().contains("selected default CDI GPU device"),
1285-
"unexpected error: {}",
1286-
err.message()
1287+
create_body
1288+
.host_config
1289+
.as_ref()
1290+
.and_then(|host_config| host_config.device_requests.as_ref())
1291+
.is_none()
12871292
);
12881293
}
12891294

@@ -1362,8 +1367,11 @@ fn build_container_create_body_rejects_empty_cdi_devices() {
13621367
fn driver_default_gpu_selection_consumes_distinct_devices_for_creates() {
13631368
let mut config = runtime_config();
13641369
config.supports_gpu = true;
1365-
config.cdi_gpu_inventory = CdiGpuInventory::new(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]);
13661370
let driver = test_driver_with_config(config);
1371+
driver.gpu_selector.refresh(
1372+
CdiGpuInventory::new(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]),
1373+
false,
1374+
);
13671375
let mut first_sandbox = test_sandbox();
13681376
first_sandbox.id = "sbx-first".to_string();
13691377
first_sandbox.name = "first".to_string();
@@ -1375,27 +1383,30 @@ fn driver_default_gpu_selection_consumes_distinct_devices_for_creates() {
13751383

13761384
DockerComputeDriver::validate_sandbox(&first_sandbox, &driver.config).unwrap();
13771385
assert_eq!(
1378-
driver.peek_default_gpu_device(&first_sandbox).unwrap(),
1379-
Some("nvidia.com/gpu=0".to_string())
1386+
driver.gpu_selector.peek_device_ids(1).unwrap(),
1387+
vec!["nvidia.com/gpu=0".to_string()]
13801388
);
1381-
let first_device = driver.next_default_gpu_device(&first_sandbox).unwrap();
1382-
let first_create_body = build_container_create_body_with_default(
1389+
let first_devices = driver.gpu_selector.next_device_ids(1).unwrap();
1390+
let driver_config = DockerSandboxDriverConfig::default();
1391+
let first_create_body = build_container_create_body_with_gpu_devices(
13831392
&first_sandbox,
13841393
&driver.config,
1385-
first_device.as_deref(),
1394+
&driver_config,
1395+
Some(&first_devices),
13861396
)
13871397
.unwrap();
13881398

13891399
DockerComputeDriver::validate_sandbox(&second_sandbox, &driver.config).unwrap();
13901400
assert_eq!(
1391-
driver.peek_default_gpu_device(&second_sandbox).unwrap(),
1392-
Some("nvidia.com/gpu=1".to_string())
1401+
driver.gpu_selector.peek_device_ids(1).unwrap(),
1402+
vec!["nvidia.com/gpu=1".to_string()]
13931403
);
1394-
let second_device = driver.next_default_gpu_device(&second_sandbox).unwrap();
1395-
let second_create_body = build_container_create_body_with_default(
1404+
let second_devices = driver.gpu_selector.next_device_ids(1).unwrap();
1405+
let second_create_body = build_container_create_body_with_gpu_devices(
13961406
&second_sandbox,
13971407
&driver.config,
1398-
second_device.as_deref(),
1408+
&driver_config,
1409+
Some(&second_devices),
13991410
)
14001411
.unwrap();
14011412

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use kube::{Client, Error as KubeError};
1717
use openshell_core::driver_utils::{
1818
LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, SUPERVISOR_IMAGE_BINARY_PATH,
1919
};
20-
use openshell_core::gpu::driver_gpu_requirements;
20+
use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count};
2121
use openshell_core::progress::{
2222
PROGRESS_STEP_PULLING_IMAGE, PROGRESS_STEP_REQUESTING_SANDBOX, PROGRESS_STEP_STARTING_SANDBOX,
2323
format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail,
@@ -84,7 +84,6 @@ const SANDBOX_VERSION: &str = "v1alpha1";
8484
pub const SANDBOX_KIND: &str = "Sandbox";
8585

8686
const GPU_RESOURCE_NAME: &str = "nvidia.com/gpu";
87-
const GPU_RESOURCE_QUANTITY: &str = "1";
8887
const SPIFFE_WORKLOAD_API_VOLUME_NAME: &str = "spiffe-workload-api";
8988

9089
// This POC treats the selected Struct as a driver-local typed schema. Once the
@@ -654,12 +653,8 @@ impl KubernetesComputeDriver {
654653
fn validate_gpu_request(
655654
gpu_requirements: Option<&GpuResourceRequirements>,
656655
) -> Result<(), tonic::Status> {
657-
if gpu_requirements.and_then(|gpu| gpu.count) == Some(0) {
658-
return Err(tonic::Status::invalid_argument(
659-
"gpu count must be greater than 0",
660-
));
661-
}
662-
656+
let _ =
657+
effective_driver_gpu_count(gpu_requirements).map_err(tonic::Status::invalid_argument)?;
663658
Ok(())
664659
}
665660

@@ -1704,10 +1699,7 @@ fn container_resources(
17041699
}
17051700

17061701
if let Some(gpu) = gpu_requirements {
1707-
let quantity = gpu.count.map_or_else(
1708-
|| GPU_RESOURCE_QUANTITY.to_string(),
1709-
|count| count.to_string(),
1710-
);
1702+
let quantity = gpu.count.unwrap_or(1).to_string();
17111703
apply_gpu_limit(&mut resources, &quantity);
17121704
}
17131705
if resources.as_object().is_some_and(serde_json::Map::is_empty) {
@@ -2416,7 +2408,7 @@ mod tests {
24162408
);
24172409
assert_eq!(
24182410
pod_template["spec"]["containers"][0]["resources"]["limits"][GPU_RESOURCE_NAME],
2419-
serde_json::json!(GPU_RESOURCE_QUANTITY)
2411+
serde_json::json!("1")
24202412
);
24212413
}
24222414

@@ -2704,7 +2696,7 @@ mod tests {
27042696
assert_eq!(limits["cpu"], serde_json::json!("2"));
27052697
assert_eq!(
27062698
limits[GPU_RESOURCE_NAME],
2707-
serde_json::json!(GPU_RESOURCE_QUANTITY)
2699+
serde_json::json!("1")
27082700
);
27092701
}
27102702

‎crates/openshell-driver-podman/src/container.rs‎

Lines changed: 35 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,8 @@
55
66
use crate::config::PodmanComputeConfig;
77
use openshell_core::ComputeDriverError;
8-
use openshell_core::gpu::{
9-
cdi_gpu_device_ids, driver_gpu_requirements, validate_specific_gpu_device_request,
10-
};
8+
#[cfg(test)]
9+
use openshell_core::gpu::{driver_gpu_requirements, validate_specific_gpu_device_request};
1110
use openshell_core::proto::compute::v1::{DriverSandbox, DriverSandboxTemplate};
1211
use openshell_core::proto_struct::deserialize_optional_non_empty_string_list;
1312
use openshell_core::{driver_mounts, proto_struct};
@@ -478,37 +477,6 @@ fn podman_pids_limit(value: i64) -> Option<i64> {
478477
if value > 0 { Some(value) } else { None }
479478
}
480479

481-
/// Build CDI GPU device list if GPU is requested.
482-
fn build_devices(
483-
sandbox: &DriverSandbox,
484-
selected_default_device: Option<&str>,
485-
) -> Result<Option<Vec<LinuxDevice>>, ComputeDriverError> {
486-
let Some(spec) = sandbox.spec.as_ref() else {
487-
return Ok(None);
488-
};
489-
let cdi_devices = PodmanSandboxDriverConfig::from_sandbox(sandbox)?
490-
.cdi_devices
491-
.unwrap_or_default();
492-
let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref());
493-
validate_specific_gpu_device_request(
494-
gpu_requirements,
495-
&cdi_devices,
496-
"driver_config.cdi_devices",
497-
)
498-
.map_err(ComputeDriverError::InvalidArgument)?;
499-
500-
cdi_gpu_device_ids(gpu_requirements, &cdi_devices, selected_default_device)
501-
.map(|device_ids| {
502-
device_ids.map(|device_ids| {
503-
device_ids
504-
.into_iter()
505-
.map(|path| LinuxDevice { path })
506-
.collect()
507-
})
508-
})
509-
.map_err(|err| ComputeDriverError::Precondition(err.to_string()))
510-
}
511-
512480
pub fn podman_driver_volume_mount_sources(
513481
sandbox: &DriverSandbox,
514482
enable_bind_mounts: bool,
@@ -799,14 +767,30 @@ pub fn try_build_container_spec_with_token(
799767
config: &PodmanComputeConfig,
800768
token_host_path: Option<&Path>,
801769
) -> Result<Value, ComputeDriverError> {
802-
build_container_spec_with_token_and_gpu_default(sandbox, config, token_host_path, None)
770+
let driver_config = PodmanSandboxDriverConfig::from_sandbox(sandbox)?;
771+
let gpu_requirements = sandbox
772+
.spec
773+
.as_ref()
774+
.and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref()));
775+
let cdi_devices = if let Some(cdi_devices) = driver_config.cdi_devices.as_ref() {
776+
validate_specific_gpu_device_request(
777+
gpu_requirements,
778+
cdi_devices,
779+
"driver_config.cdi_devices",
780+
)
781+
.map_err(ComputeDriverError::InvalidArgument)?;
782+
Some(cdi_devices.as_slice())
783+
} else {
784+
None
785+
};
786+
build_container_spec_with_token_and_gpu_devices(sandbox, config, token_host_path, cdi_devices)
803787
}
804788

805-
pub fn build_container_spec_with_token_and_gpu_default(
789+
pub fn build_container_spec_with_token_and_gpu_devices(
806790
sandbox: &DriverSandbox,
807791
config: &PodmanComputeConfig,
808792
token_host_path: Option<&Path>,
809-
selected_default_device: Option<&str>,
793+
gpu_device_ids: Option<&[String]>,
810794
) -> Result<Value, ComputeDriverError> {
811795
let image = resolve_image(sandbox, config);
812796
let name = container_name(&sandbox.name);
@@ -817,7 +801,13 @@ pub fn build_container_spec_with_token_and_gpu_default(
817801
let resource_limits = build_resource_limits(sandbox, config);
818802
let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts)
819803
.map_err(ComputeDriverError::InvalidArgument)?;
820-
let devices = build_devices(sandbox, selected_default_device)?;
804+
let devices = gpu_device_ids.map(|device_ids| {
805+
device_ids
806+
.iter()
807+
.cloned()
808+
.map(|path| LinuxDevice { path })
809+
.collect()
810+
});
821811

822812
// Network configuration -- always bridge mode.
823813
// Matches libpod's network spec format `{name: {opts}}`; the unit-struct
@@ -1275,11 +1265,12 @@ mod tests {
12751265
..Default::default()
12761266
});
12771267
let config = test_config();
1278-
let spec = build_container_spec_with_token_and_gpu_default(
1268+
let gpu_devices = vec!["nvidia.com/gpu=1".to_string()];
1269+
let spec = build_container_spec_with_token_and_gpu_devices(
12791270
&sandbox,
12801271
&config,
12811272
None,
1282-
Some("nvidia.com/gpu=1"),
1273+
Some(&gpu_devices),
12831274
)
12841275
.unwrap();
12851276

@@ -1290,7 +1281,7 @@ mod tests {
12901281
}
12911282

12921283
#[test]
1293-
fn container_spec_rejects_missing_default_cdi_device() {
1284+
fn container_spec_omits_devices_without_resolved_default_cdi_devices() {
12941285
use openshell_core::proto::compute::v1::DriverSandboxSpec;
12951286

12961287
let mut sandbox = test_sandbox("test-id", "test-name");
@@ -1300,11 +1291,10 @@ mod tests {
13001291
});
13011292
let config = test_config();
13021293

1303-
let err = build_container_spec_with_token_and_gpu_default(&sandbox, &config, None, None)
1304-
.unwrap_err();
1294+
let spec =
1295+
build_container_spec_with_token_and_gpu_devices(&sandbox, &config, None, None).unwrap();
13051296

1306-
assert!(matches!(err, ComputeDriverError::Precondition(_)));
1307-
assert!(err.to_string().contains("selected default CDI GPU device"));
1297+
assert!(spec.get("devices").is_none());
13081298
}
13091299

13101300
#[test]

0 commit comments

Comments
 (0)