From 1eec0a4d326a81242adce34acb893bd42a045a7e Mon Sep 17 00:00:00 2001 From: Raphael Glon Date: Thu, 18 Dec 2025 16:02:30 +0100 Subject: [PATCH 1/2] fix(num_devices): fix num_shard/num device auto compute when NVIDIA_VISIBLE_DEVICES == "all" the computed num_shards was always 1 in this case, no matter what Signed-off-by: Raphael Glon --- launcher/src/main.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index c727623ce47..ded671e436d 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -72,11 +72,11 @@ fn vram_maximum( if let Some(vram) = available.checked_sub(model) { let tokens_allowed = vram / token_vram; tracing::debug!( - "Available vram {}: model needs {}, every tokens requires {}, maximum allocatable tokens {tokens_allowed}", - human_size(available, "B"), - human_size(model, "B"), - human_size(token_vram, "B"), - ); + "Available vram {}: model needs {}, every tokens requires {}, maximum allocatable tokens {tokens_allowed}", + human_size(available, "B"), + human_size(model, "B"), + human_size(token_vram, "B"), + ); Some(tokens_allowed) } else { tracing::warn!( @@ -1274,8 +1274,7 @@ fn num_cuda_devices() -> Option { String::from_utf8_lossy(&output.stdout) .lines() .filter(|line| !line.trim().is_empty()) - .count() - .to_string() + .collect::>().join(",") } else { devices } From 60a2a4b2985d9b8fde1278a425dfdc5cdd3456f5 Mon Sep 17 00:00:00 2001 From: Raphael Glon Date: Thu, 18 Dec 2025 16:19:39 +0100 Subject: [PATCH 2/2] fix(num_shard): make TGI compliant with nvidia-container-toolkit in cdi mode Signed-off-by: Raphael Glon --- launcher/src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index ded671e436d..09b953176a8 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1264,7 +1264,12 @@ fn num_cuda_devices() -> Option { Ok(devices) => devices, Err(_) => match env::var("NVIDIA_VISIBLE_DEVICES") { Ok(devices) => { - if devices.trim() == "all" { + // NVIDIA_VISIBLE_DEVICES is always set when not specified and the nvidia container runtime is + // in (jit-)cdi mode (since 1.14) + // nvidia container runtime default mode switched from legacy to cdi mode from 1.18 on + // Let's handle the void case as all here + // See: https://github.com/NVIDIA/nvidia-container-toolkit + if ["all", "void"].contains(&devices.trim()) { // Count the number of all GPUs via nvidia-smi let output = Command::new("nvidia-smi") .args(["--query-gpu=uuid", "--format=csv,noheader"])