Skip to content

Commit 7cebf5f

Browse files
[ci] cap kernel wheel build parallelism to avoid runner OOM (#1483)
1 parent dd0f4b6 commit 7cebf5f

1 file changed

Lines changed: 10 additions & 0 deletions

File tree

.github/workflows/publish-kernel.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,12 +148,22 @@ jobs:
148148
# The CUDA 13 (cu130) leg additionally targets Blackwell sm_120a and builds the
149149
# attn_qat_infer FP4 inference kernels (they need CUDA Toolkit 12.8+, so the cu126 leg
150150
# stays Hopper-only and ships without them).
151+
# Cap Ninja build concurrency to fit the 16 GB standard runner. CUTLASS FP4
152+
# and ThunderKittens template TUs each need several GB; Ninja's default
153+
# (cores+2 ≈ 6 parallel jobs) exhausts RAM and the host OOM-kills the build
154+
# (SIGTERM / exit 143, surfaced as "runner lost communication").
151155
if [ "${{ matrix.torch-cuda.torch-cuda-short }}" = "cu130" ]; then
152156
export TORCH_CUDA_ARCH_LIST="9.0a;12.0a"
153157
export CMAKE_ARGS="${CMAKE_ARGS:-} -DFASTVIDEO_KERNEL_BUILD_TK=ON -DFASTVIDEO_KERNEL_BUILD_ATTN_QAT_INFER=ON -DCMAKE_CUDA_ARCHITECTURES=90a"
158+
# A single FP4 TU (attn_qat_infer) can use ~8-12 GB on its own, so serialize.
159+
export CMAKE_BUILD_PARALLEL_LEVEL=1
154160
else
155161
export TORCH_CUDA_ARCH_LIST="9.0a"
156162
export CMAKE_ARGS="${CMAKE_ARGS:-} -DFASTVIDEO_KERNEL_BUILD_TK=ON -DCMAKE_CUDA_ARCHITECTURES=90a"
163+
# No FP4 here; only the two TK TUs are heavy (single-arch) and they fit
164+
# side by side. Uncapped (cores+2=6) has built but occasionally OOMs when
165+
# the light TUs pile on; -j4 overlaps them safely (~12-14 GB peak).
166+
export CMAKE_BUILD_PARALLEL_LEVEL=4
157167
fi
158168
159169
# Build standard wheel (no local version suffix) for PyPI

0 commit comments

Comments
 (0)