Skip to content

Commit f982cda

Browse files
authored
Add gfx1100 ROCm CI for AOTI (pytorch#21957)
Add an opt-in, path-filtered ROCm job on the gfx950 runner. Build the HIP runtime and native runner, exercise targeted Python and C++ coverage, verify generated AOTI code and HIP-only linkage, and provide a generic ROCm LLM preset.
1 parent b87b417 commit f982cda

6 files changed

Lines changed: 334 additions & 11 deletions

File tree

‎.ci/scripts/test-rocm-aoti.sh‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -euo pipefail
9+
10+
ROCM_VERSION="${ROCM_VERSION:-7.1}"
11+
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
12+
PYTORCH_ROCM_INDEX="${PYTORCH_ROCM_INDEX:-https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}}"
13+
TORCHAO_ROCM_WHEEL_BASE="${TORCHAO_ROCM_WHEEL_BASE:-https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}}"
14+
ROCM_CI_TMP_ROOT="${RUNNER_TEMP:-/tmp}"
15+
mkdir -p "${ROCM_CI_TMP_ROOT}" 2>/dev/null || ROCM_CI_TMP_ROOT=/tmp
16+
ROCM_CI_TMPDIR="$(mktemp -d "${ROCM_CI_TMP_ROOT}/executorch-rocm-ci.XXXXXX")"
17+
trap 'rm -rf "${ROCM_CI_TMPDIR}"' EXIT
18+
19+
export ROCM_PATH
20+
export HIP_VISIBLE_DEVICES=0
21+
export CUDA_VISIBLE_DEVICES=0
22+
export TORCHINDUCTOR_CACHE_DIR="${ROCM_CI_TMPDIR}/inductor-cache"
23+
export TORCHINDUCTOR_COMPILE_THREADS=1
24+
25+
read -r TORCH_VERSION TORCHAO_VERSION < <(
26+
python - <<'PY'
27+
from install_requirements import TORCHAO_NIGHTLY_VERSION
28+
from torch_pin import TORCH_VERSION
29+
30+
print(TORCH_VERSION, TORCHAO_NIGHTLY_VERSION)
31+
PY
32+
)
33+
# TorchAO ROCm wheels are not exposed by the per-version pip index.
34+
TORCHAO_WHEEL="${TORCHAO_ROCM_WHEEL_BASE}/torchao-${TORCHAO_VERSION}"
35+
TORCHAO_WHEEL+="%2Brocm${ROCM_VERSION}-cp310-abi3-manylinux_2_28_x86_64.whl"
36+
python -m pip install "torch==${TORCH_VERSION}" \
37+
--index-url "${PYTORCH_ROCM_INDEX}"
38+
python -m pip install -r requirements-dev.txt \
39+
"${TORCHAO_WHEEL}"
40+
EXECUTORCH_BUILD_MINIMAL=1 \
41+
python -m pip install --editable . --no-build-isolation
42+
43+
if ! command -v conda >/dev/null; then
44+
echo "The ROCm CI image must provide conda for its C++ runtime libraries"
45+
exit 1
46+
fi
47+
conda install -y -c conda-forge 'libstdcxx-ng>=12'
48+
49+
python - <<'PY'
50+
import torch
51+
import torchao
52+
import triton
53+
54+
assert torch.version.hip is not None, "PyTorch is not a ROCm build"
55+
assert torch.version.cuda is None, "PyTorch unexpectedly reports a CUDA runtime"
56+
assert torch.cuda.is_available(), "No AMD GPU is visible through PyTorch"
57+
assert "+rocm" in torchao.__version__, "TorchAO is not a ROCm build"
58+
59+
device = torch.cuda.get_device_properties(0)
60+
arch = device.gcnArchName.split(":", 1)[0]
61+
assert arch == "gfx950", f"Expected gfx950, got {device.gcnArchName}"
62+
assert device.warp_size == 64, f"Expected wave64, got {device.warp_size}"
63+
assert arch in torch.cuda.get_arch_list(), (
64+
f"{arch} is not supported by this PyTorch build: {torch.cuda.get_arch_list()}"
65+
)
66+
67+
print("PyTorch:", torch.__version__)
68+
print("TorchAO:", torchao.__version__)
69+
print("ROCm:", torch.version.hip)
70+
print("Triton:", triton.__version__)
71+
print("Device:", device.name, device.gcnArchName)
72+
print("Architectures:", torch.cuda.get_arch_list())
73+
PY
74+
75+
if command -v rocminfo >/dev/null; then
76+
rocminfo | sed -n '1,160p'
77+
fi
78+
if command -v rocm-smi >/dev/null; then
79+
rocm-smi --showproductname --showmeminfo vram --showuse --showtemp || true
80+
elif command -v amd-smi >/dev/null; then
81+
amd-smi static --gpu all || true
82+
fi
83+
84+
TORCH_CMAKE_PREFIX="$(python -c 'import torch; print(torch.utils.cmake_prefix_path)')"
85+
export CMAKE_PREFIX_PATH="${TORCH_CMAKE_PREFIX}"
86+
87+
cmake --preset llm-release-rocm -DEXECUTORCH_BUILD_TESTS=ON
88+
cmake --build cmake-out-rocm-llm \
89+
--target executor_runner test_cuda_allocator test_cuda_mutable_state \
90+
--parallel "$(nproc)"
91+
92+
PYTHON_PREFIX="$(python -c 'import sys; print(sys.prefix)')"
93+
export LD_LIBRARY_PATH="${PYTHON_PREFIX}/lib:${PWD}/cmake-out-rocm-llm/backends/cuda:${PWD}/cmake-out-rocm-llm/extension/cuda:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
94+
95+
ctest --test-dir cmake-out-rocm-llm \
96+
-R 'test_cuda_(allocator|mutable_state)' \
97+
-V
98+
99+
# Targeted builds leave the shim in the build tree rather than install lib/.
100+
for ROCM_CI_BINARY in \
101+
cmake-out-rocm-llm/backends/cuda/libaoti_cuda_shims.so \
102+
cmake-out-rocm-llm/executor_runner; do
103+
ROCM_CI_LINKS="$(ldd "${ROCM_CI_BINARY}")"
104+
printf '%s\n' "${ROCM_CI_LINKS}"
105+
if grep -q 'not found' <<<"${ROCM_CI_LINKS}"; then
106+
echo "Missing runtime dependency in ${ROCM_CI_BINARY}"
107+
exit 1
108+
fi
109+
if grep -q 'libcudart' <<<"${ROCM_CI_LINKS}"; then
110+
echo "Unexpected CUDA runtime dependency in ${ROCM_CI_BINARY}"
111+
exit 1
112+
fi
113+
if ! grep -q 'libamdhip64' <<<"${ROCM_CI_LINKS}"; then
114+
echo "HIP runtime dependency missing from ${ROCM_CI_BINARY}"
115+
exit 1
116+
fi
117+
done
118+
119+
if ! grep -q '^EXECUTORCH_BUILD_CUDA:BOOL=OFF$' \
120+
cmake-out-rocm-llm/CMakeCache.txt; then
121+
echo "ROCm CI unexpectedly enabled the CUDA build"
122+
exit 1
123+
fi
124+
if ! grep -q '^EXECUTORCH_BUILD_ROCM:BOOL=ON$' \
125+
cmake-out-rocm-llm/CMakeCache.txt; then
126+
echo "ROCm CI did not enable the ROCm build"
127+
exit 1
128+
fi
129+
if ! grep -q 'find_dependency(hip CONFIG)' \
130+
cmake-out-rocm-llm/executorch-backend-dependencies.cmake; then
131+
echo "ROCm package metadata does not declare its HIP dependency"
132+
exit 1
133+
fi
134+
if grep -q 'find_dependency(CUDAToolkit)' \
135+
cmake-out-rocm-llm/executorch-backend-dependencies.cmake; then
136+
echo "ROCm package metadata unexpectedly depends on CUDAToolkit"
137+
exit 1
138+
fi
139+
140+
export EXECUTORCH_EXECUTOR_RUNNER="${PWD}/cmake-out-rocm-llm/executor_runner"
141+
ROCM_EXCLUDED_TESTS=(
142+
# This focused job does not install the optional flash-linear-attention package.
143+
--ignore=backends/cuda/tests/test_chunk_gated_delta_rule.py
144+
# Defer specialized SDPA suites until their CI cost is measured.
145+
--ignore=backends/cuda/tests/test_tq4_sdpa.py
146+
--ignore=backends/cuda/tests/test_triton_sdpa.py
147+
--ignore=backends/cuda/tests/test_triton_sdpa_splitk.py
148+
# This test process core-dumps on ROCm instead of reporting a failure.
149+
--ignore=backends/cuda/tests/test_triton_sdpa_nan.py
150+
)
151+
python -m pytest -v -o 'addopts=' \
152+
"${ROCM_EXCLUDED_TESTS[@]}" \
153+
backends/cuda/tests \
154+
backends/cuda/passes/tests
155+
156+
ROCM_POINTWISE_DIR="${ROCM_CI_TMPDIR}/pointwise"
157+
python -m examples.cuda.scripts.export_amd_pointwise \
158+
--output-dir "${ROCM_POINTWISE_DIR}"

‎.github/pytorch-probot.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ ciflow_push_tags:
77
- ciflow/cuda-perf
88
- ciflow/metal
99
- ciflow/mlx
10+
- ciflow/rocm
1011
- ciflow/nightly
1112
- ciflow/trunk
1213
- ciflow/binaries

‎.github/workflows/rocm.yml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Test ExecuTorch AOTI ROCm support.
2+
# Use gfx950 for functional coverage and reserve scarce gfx1100 RDNA runners
3+
# for a future focused performance job.
4+
5+
name: Test ROCm AOTI
6+
7+
on:
8+
push:
9+
branches:
10+
- main
11+
- release/*
12+
tags:
13+
- ciflow/rocm/*
14+
pull_request:
15+
paths:
16+
- .github/workflows/rocm.yml
17+
- .github/workflows/_ci-run-decision.yml
18+
- .github/workflows/_get-changed-files.yml
19+
- .ci/scripts/test-rocm-aoti.sh
20+
- CMakeLists.txt
21+
- CMakePresets.json
22+
- install_requirements.py
23+
- torch_pin.py
24+
- backends/aoti/**
25+
- '!backends/aoti/**.md'
26+
- backends/cuda/**
27+
- '!backends/cuda/**.md'
28+
- extension/cuda/**
29+
- examples/cuda/**
30+
- '!examples/cuda/**.md'
31+
workflow_dispatch:
32+
33+
concurrency:
34+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
35+
cancel-in-progress: false
36+
37+
permissions:
38+
contents: read
39+
40+
jobs:
41+
changed-files:
42+
name: Get changed files
43+
uses: ./.github/workflows/_get-changed-files.yml
44+
with:
45+
include-push-diff: true
46+
47+
run-decision:
48+
name: CI run decision
49+
uses: ./.github/workflows/_ci-run-decision.yml
50+
51+
aoti-run-decision:
52+
name: AOTI CI run decision
53+
needs: [changed-files, run-decision]
54+
runs-on: ubuntu-latest
55+
outputs:
56+
run-aoti: ${{ steps.decide.outputs.run-aoti }}
57+
steps:
58+
- name: Select AOTI runs
59+
id: decide
60+
env:
61+
CHANGED_FILES: ${{ needs.changed-files.outputs.changed-files }}
62+
FULL_RUN: ${{ needs.run-decision.outputs.is-full-run }}
63+
run: |
64+
set -efu
65+
RUN_AOTI=false
66+
67+
if [[ "$CHANGED_FILES" == "*" || "$FULL_RUN" == "true" ]]; then
68+
RUN_AOTI=true
69+
fi
70+
71+
for path in $CHANGED_FILES; do
72+
case "$path" in
73+
backends/aoti/*.md | backends/cuda/*.md | examples/cuda/*.md)
74+
continue
75+
;;
76+
.github/workflows/rocm.yml | \
77+
.github/workflows/_ci-run-decision.yml | \
78+
.github/workflows/_get-changed-files.yml | \
79+
.ci/scripts/test-rocm-aoti.sh | \
80+
CMakeLists.txt | \
81+
CMakePresets.json | \
82+
install_requirements.py | \
83+
torch_pin.py | \
84+
backends/aoti/* | \
85+
backends/cuda/* | \
86+
extension/cuda/* | \
87+
examples/cuda/*)
88+
RUN_AOTI=true
89+
break
90+
;;
91+
esac
92+
done
93+
94+
echo "run-aoti=$RUN_AOTI" >> "$GITHUB_OUTPUT"
95+
96+
unittest-rocm-gfx950:
97+
name: unittest-rocm-gfx950-rocm${{ matrix.rocm-version }}
98+
needs: [aoti-run-decision]
99+
if: needs.aoti-run-decision.outputs.run-aoti == 'true'
100+
strategy:
101+
fail-fast: false
102+
matrix:
103+
rocm-version: ["7.1"]
104+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
105+
permissions:
106+
id-token: write
107+
contents: read
108+
with:
109+
timeout: 90
110+
no-sudo: true
111+
runner: linux.rocm.gpu.gfx950.1
112+
gpu-arch-type: rocm
113+
gpu-arch-version: ${{ matrix.rocm-version }}
114+
use-custom-docker-registry: false
115+
docker-image: pytorch/manylinux2_28-builder:rocm${{ matrix.rocm-version }}
116+
submodules: recursive
117+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
118+
script: |
119+
set -eux
120+
eval "$(conda shell.bash hook)"
121+
conda create -n executorch-rocm-ci python=3.10 -y
122+
conda activate executorch-rocm-ci
123+
ROCM_VERSION=${{ matrix.rocm-version }} bash .ci/scripts/test-rocm-aoti.sh

‎CMakePresets.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,25 @@
161161
"list": ["Linux", "Windows"]
162162
}
163163
},
164+
{
165+
"name": "llm-release-rocm",
166+
"displayName": "LLM release build with ROCm",
167+
"inherits": [
168+
"llm-release"
169+
],
170+
"binaryDir": "${sourceDir}/cmake-out-rocm-llm",
171+
"cacheVariables": {
172+
"CMAKE_INSTALL_PREFIX": "${sourceDir}/cmake-out-rocm-llm",
173+
"CMAKE_PREFIX_PATH": "$penv{ROCM_PATH};/opt/rocm;$penv{CMAKE_PREFIX_PATH}",
174+
"EXECUTORCH_BUILD_CUDA": "OFF",
175+
"EXECUTORCH_BUILD_ROCM": "ON"
176+
},
177+
"condition": {
178+
"lhs": "${hostSystemName}",
179+
"type": "equals",
180+
"rhs": "Linux"
181+
}
182+
},
164183
{
165184
"name": "llm-release-metal",
166185
"displayName": "LLM release build with Metal",
@@ -397,6 +416,16 @@
397416
],
398417
"jobs": 0
399418
},
419+
{
420+
"name": "llm-release-rocm-install",
421+
"displayName": "Build and install LLM extension release artifacts (ROCm)",
422+
"configurePreset": "llm-release-rocm",
423+
"configuration": "Release",
424+
"targets": [
425+
"install"
426+
],
427+
"jobs": 0
428+
},
400429
{
401430
"name": "llm-release-metal-install",
402431
"displayName": "Build and install LLM extension release artifacts (Metal)",
@@ -500,6 +529,20 @@
500529
}
501530
]
502531
},
532+
{
533+
"name": "llm-release-rocm",
534+
"displayName": "Configure, build and install ExecuTorch LLM extension with ROCm enabled",
535+
"steps": [
536+
{
537+
"type": "configure",
538+
"name": "llm-release-rocm"
539+
},
540+
{
541+
"type": "build",
542+
"name": "llm-release-rocm-install"
543+
}
544+
]
545+
},
503546
{
504547
"name": "llm-release-metal",
505548
"displayName": "Configure, build and install ExecuTorch LLM extension with Metal enabled",

‎backends/cuda/rocm.md‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ GPUs as device type `cuda`, so names such as `CudaBackend` and
77
`EXECUTORCH_BUILD_ROCM` is off by default, is never auto-enabled, and is
88
mutually exclusive with `EXECUTORCH_BUILD_CUDA`. It requires
99
`EXECUTORCH_BUILD_EXTENSION_TENSOR`. Execution has been validated only on
10-
MI300X (`gfx942`).
10+
MI300X (`gfx942`); CI is configured to exercise `gfx950`.
1111

1212
## Requirements and limitations
1313

@@ -18,23 +18,19 @@ MI300X (`gfx942`).
1818
- Python pybindings cannot allocate ROCm device memory; use a native runner such
1919
as `executor_runner`.
2020
- Installed CMake consumers must be able to find the HIP package.
21-
- Model runners do not yet link the ROCm backend, and there is no ROCm CI.
21+
- Model runners do not yet claim ROCm support. The gfx950 CI job covers the
22+
backend build, native runtime, AOTI export and execution, and Triton W4 tests.
2223

2324
## Build
2425

2526
```bash
26-
cmake -S . -B cmake-out-rocm \
27-
-DCMAKE_BUILD_TYPE=Release \
28-
-DCMAKE_PREFIX_PATH="$(python -c 'import torch; print(torch.utils.cmake_prefix_path)');/opt/rocm" \
29-
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
30-
-DEXECUTORCH_BUILD_ROCM=ON
31-
cmake --build cmake-out-rocm --target aoti_cuda_backend aoti_cuda_shims -j
27+
cmake --workflow --preset llm-release-rocm
3228
```
3329

3430
Verify that the build uses HIP and not the NVIDIA runtime:
3531

3632
```bash
37-
ldd cmake-out-rocm/backends/cuda/libaoti_cuda_shims.so | grep -E 'amdhip64|cudart'
33+
ldd cmake-out-rocm-llm/lib/libaoti_cuda_shims.so | grep -E 'amdhip64|cudart'
3834
```
3935

4036
A `libcudart` dependency is a configuration error.

0 commit comments

Comments
 (0)