fix(mi355x): wait up to 60 minutes for prior-job VRAM to drain (#4827)

Author: cquil11

inferencex-e2e/configs/runners.yaml

@@ -820,8 +820,8 @@ clusters:
           script: runners/srt-slurm/hooks/mi355x-amds/setup.sh
           env:
             IBDEVICES: '@fabric.rdma-devices'
-          # Covers the hook's 30-minute wait for prior-job VRAM to drain.
-          timeout-s: 2400
+          # Covers the hook's 60-minute wait for prior-job VRAM to drain.
+          timeout-s: 4200
           nodes: all
         volume-mounts:
           shared-hf-hub-cache: /hf_hub_cache/hub

inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh

@@ -8,11 +8,11 @@ bash "$(dirname "${BASH_SOURCE[0]}")/check-rdma.sh"
 # the next model load fail much later with a misleading OOM.
 # shellcheck source=runners/srt-slurm/hooks/common.sh
 source "$(dirname "${BASH_SOURCE[0]}")/../common.sh"
-# After a job ends, a node can hold 15-26 GiB per GPU for up to ~30 minutes with no
-# process attached. vLLM then fails its startup free-memory check, and SGLang
-# silently sizes a smaller mamba cache and caps max_running_requests below the
-# benchmark's concurrency. Clean nodes read 0%, so wait for that.
-wait_for_amd_gpu_clean 2 30
+# After a vLLM job ends, a node can hold 3-7% VRAM per GPU with no process attached
+# for more than 30 minutes before it drains. vLLM then fails its startup free-memory
+# check, and SGLang silently caps max_running_requests below the benchmark's
+# concurrency. Clean nodes read 0%, so wait for that.
+wait_for_amd_gpu_clean 2 60
 
 # Some MI355X experiments reserve large 2 MiB HugeTLB pools and leave the
 # reservation behind after their Slurm allocation exits. Those free hugepages