From f3598fd9f726c9fbda76848cc65d301d216fead0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 4 Nov 2026 22:05:13 -0600 Subject: [PATCH 1/2] fix(gb300): exclude c003 until its local NVMe is checked --- inferencex-e2e/configs/runners.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index ffeac20d58..9350938bb3 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -647,6 +647,9 @@ clusters: partitions: [batch_1, batch_3] account: benchmark exclusive: false + # 2026-11-05: c003's local NVMe extracted containers with zero-filled blocks (libcuda stub); + # its staged model copy is suspect too. Remove after ops checks the disk. + exclude: [im-gb300-r01-c003] volumes: hf-hub-cache: {path: /data/home/sa-shared/gharunners/hf-hub-cache} aiperf-cache: {path: /data/home/sa-shared/gharunners/ai-perf-cache} From 21a08ebb8f7c2c68d71fa097fcd4a8555b0fc92a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 4 Nov 2026 22:07:31 -0600 Subject: [PATCH 2/2] fix(gb300): also exclude c008, which read back empty Triton cache files --- inferencex-e2e/configs/runners.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index 9350938bb3..9632c2baa8 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -647,9 +647,9 @@ clusters: partitions: [batch_1, batch_3] account: benchmark exclusive: false - # 2026-11-05: c003's local NVMe extracted containers with zero-filled blocks (libcuda stub); - # its staged model copy is suspect too. Remove after ops checks the disk. - exclude: [im-gb300-r01-c003] + # 2026-11-05: c003's local NVMe extracted containers with zero-filled blocks (libcuda stub), + # and c008 twice read back empty Triton cache files. Remove after ops checks both disks. + exclude: [im-gb300-r01-c003, im-gb300-r01-c008] volumes: hf-hub-cache: {path: /data/home/sa-shared/gharunners/hf-hub-cache} aiperf-cache: {path: /data/home/sa-shared/gharunners/ai-perf-cache}