From a0f9a1df14d89876624a4f0941eed377aec4bd93 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 4 Nov 2026 19:20:00 -0600 Subject: [PATCH] fix(mi355x): wait for prior-job VRAM to drain below 5% before setup --- inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh b/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh index 2c281bf90b..433926413f 100755 --- a/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh +++ b/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh @@ -8,7 +8,10 @@ bash "$(dirname "${BASH_SOURCE[0]}")/check-rdma.sh" # the next model load fail much later with a misleading OOM. # shellcheck source=runners/srt-slurm/hooks/common.sh source "$(dirname "${BASH_SOURCE[0]}")/../common.sh" -wait_for_amd_gpu_clean +# After a job ends, a node can hold 15-26 GiB per GPU for minutes with no process +# attached; 10% passes that, and a TP8 vLLM server at 0.90 then fails its startup +# free-memory check. 5% leaves room for the engine's own context. +wait_for_amd_gpu_clean 5 # Some MI355X experiments reserve large 2 MiB HugeTLB pools and leave the # reservation behind after their Slurm allocation exits. Those free hugepages