|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | +# All rights reserved. |
| 4 | +# |
| 5 | +# This source code is licensed under the BSD-style license found in the |
| 6 | +# LICENSE file in the root directory of this source tree. |
| 7 | + |
| 8 | +set -euo pipefail |
| 9 | + |
| 10 | +ROCM_VERSION="${ROCM_VERSION:-7.1}" |
| 11 | +ROCM_PATH="${ROCM_PATH:-/opt/rocm}" |
| 12 | +PYTORCH_ROCM_INDEX="${PYTORCH_ROCM_INDEX:-https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}}" |
| 13 | +TORCHAO_ROCM_WHEEL_BASE="${TORCHAO_ROCM_WHEEL_BASE:-https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}}" |
| 14 | +ROCM_CI_TMP_ROOT="${RUNNER_TEMP:-/tmp}" |
| 15 | +mkdir -p "${ROCM_CI_TMP_ROOT}" 2>/dev/null || ROCM_CI_TMP_ROOT=/tmp |
| 16 | +ROCM_CI_TMPDIR="$(mktemp -d "${ROCM_CI_TMP_ROOT}/executorch-rocm-ci.XXXXXX")" |
| 17 | +trap 'rm -rf "${ROCM_CI_TMPDIR}"' EXIT |
| 18 | + |
| 19 | +export ROCM_PATH |
| 20 | +export HIP_VISIBLE_DEVICES=0 |
| 21 | +export CUDA_VISIBLE_DEVICES=0 |
| 22 | +export TORCHINDUCTOR_CACHE_DIR="${ROCM_CI_TMPDIR}/inductor-cache" |
| 23 | +export TORCHINDUCTOR_COMPILE_THREADS=1 |
| 24 | + |
| 25 | +read -r TORCH_VERSION TORCHAO_VERSION < <( |
| 26 | + python - <<'PY' |
| 27 | +from install_requirements import TORCHAO_NIGHTLY_VERSION |
| 28 | +from torch_pin import TORCH_VERSION |
| 29 | + |
| 30 | +print(TORCH_VERSION, TORCHAO_NIGHTLY_VERSION) |
| 31 | +PY |
| 32 | +) |
| 33 | +# TorchAO ROCm wheels are not exposed by the per-version pip index. |
| 34 | +TORCHAO_WHEEL="${TORCHAO_ROCM_WHEEL_BASE}/torchao-${TORCHAO_VERSION}" |
| 35 | +TORCHAO_WHEEL+="%2Brocm${ROCM_VERSION}-cp310-abi3-manylinux_2_28_x86_64.whl" |
| 36 | +python -m pip install "torch==${TORCH_VERSION}" \ |
| 37 | + --index-url "${PYTORCH_ROCM_INDEX}" |
| 38 | +python -m pip install -r requirements-dev.txt \ |
| 39 | + "${TORCHAO_WHEEL}" |
| 40 | +EXECUTORCH_BUILD_MINIMAL=1 \ |
| 41 | + python -m pip install --editable . --no-build-isolation |
| 42 | + |
| 43 | +if ! command -v conda >/dev/null; then |
| 44 | + echo "The ROCm CI image must provide conda for its C++ runtime libraries" |
| 45 | + exit 1 |
| 46 | +fi |
| 47 | +conda install -y -c conda-forge 'libstdcxx-ng>=12' |
| 48 | + |
| 49 | +python - <<'PY' |
| 50 | +import torch |
| 51 | +import torchao |
| 52 | +import triton |
| 53 | + |
| 54 | +assert torch.version.hip is not None, "PyTorch is not a ROCm build" |
| 55 | +assert torch.version.cuda is None, "PyTorch unexpectedly reports a CUDA runtime" |
| 56 | +assert torch.cuda.is_available(), "No AMD GPU is visible through PyTorch" |
| 57 | +assert "+rocm" in torchao.__version__, "TorchAO is not a ROCm build" |
| 58 | + |
| 59 | +device = torch.cuda.get_device_properties(0) |
| 60 | +arch = device.gcnArchName.split(":", 1)[0] |
| 61 | +assert arch == "gfx950", f"Expected gfx950, got {device.gcnArchName}" |
| 62 | +assert device.warp_size == 64, f"Expected wave64, got {device.warp_size}" |
| 63 | +assert arch in torch.cuda.get_arch_list(), ( |
| 64 | + f"{arch} is not supported by this PyTorch build: {torch.cuda.get_arch_list()}" |
| 65 | +) |
| 66 | + |
| 67 | +print("PyTorch:", torch.__version__) |
| 68 | +print("TorchAO:", torchao.__version__) |
| 69 | +print("ROCm:", torch.version.hip) |
| 70 | +print("Triton:", triton.__version__) |
| 71 | +print("Device:", device.name, device.gcnArchName) |
| 72 | +print("Architectures:", torch.cuda.get_arch_list()) |
| 73 | +PY |
| 74 | + |
| 75 | +if command -v rocminfo >/dev/null; then |
| 76 | + rocminfo | sed -n '1,160p' |
| 77 | +fi |
| 78 | +if command -v rocm-smi >/dev/null; then |
| 79 | + rocm-smi --showproductname --showmeminfo vram --showuse --showtemp || true |
| 80 | +elif command -v amd-smi >/dev/null; then |
| 81 | + amd-smi static --gpu all || true |
| 82 | +fi |
| 83 | + |
| 84 | +TORCH_CMAKE_PREFIX="$(python -c 'import torch; print(torch.utils.cmake_prefix_path)')" |
| 85 | +export CMAKE_PREFIX_PATH="${TORCH_CMAKE_PREFIX}" |
| 86 | + |
| 87 | +cmake --preset llm-release-rocm -DEXECUTORCH_BUILD_TESTS=ON |
| 88 | +cmake --build cmake-out-rocm-llm \ |
| 89 | + --target executor_runner test_cuda_allocator test_cuda_mutable_state \ |
| 90 | + --parallel "$(nproc)" |
| 91 | + |
| 92 | +PYTHON_PREFIX="$(python -c 'import sys; print(sys.prefix)')" |
| 93 | +export LD_LIBRARY_PATH="${PYTHON_PREFIX}/lib:${PWD}/cmake-out-rocm-llm/backends/cuda:${PWD}/cmake-out-rocm-llm/extension/cuda:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}" |
| 94 | + |
| 95 | +ctest --test-dir cmake-out-rocm-llm \ |
| 96 | + -R 'test_cuda_(allocator|mutable_state)' \ |
| 97 | + -V |
| 98 | + |
| 99 | +# Targeted builds leave the shim in the build tree rather than install lib/. |
| 100 | +for ROCM_CI_BINARY in \ |
| 101 | + cmake-out-rocm-llm/backends/cuda/libaoti_cuda_shims.so \ |
| 102 | + cmake-out-rocm-llm/executor_runner; do |
| 103 | + ROCM_CI_LINKS="$(ldd "${ROCM_CI_BINARY}")" |
| 104 | + printf '%s\n' "${ROCM_CI_LINKS}" |
| 105 | + if grep -q 'not found' <<<"${ROCM_CI_LINKS}"; then |
| 106 | + echo "Missing runtime dependency in ${ROCM_CI_BINARY}" |
| 107 | + exit 1 |
| 108 | + fi |
| 109 | + if grep -q 'libcudart' <<<"${ROCM_CI_LINKS}"; then |
| 110 | + echo "Unexpected CUDA runtime dependency in ${ROCM_CI_BINARY}" |
| 111 | + exit 1 |
| 112 | + fi |
| 113 | + if ! grep -q 'libamdhip64' <<<"${ROCM_CI_LINKS}"; then |
| 114 | + echo "HIP runtime dependency missing from ${ROCM_CI_BINARY}" |
| 115 | + exit 1 |
| 116 | + fi |
| 117 | +done |
| 118 | + |
| 119 | +if ! grep -q '^EXECUTORCH_BUILD_CUDA:BOOL=OFF$' \ |
| 120 | + cmake-out-rocm-llm/CMakeCache.txt; then |
| 121 | + echo "ROCm CI unexpectedly enabled the CUDA build" |
| 122 | + exit 1 |
| 123 | +fi |
| 124 | +if ! grep -q '^EXECUTORCH_BUILD_ROCM:BOOL=ON$' \ |
| 125 | + cmake-out-rocm-llm/CMakeCache.txt; then |
| 126 | + echo "ROCm CI did not enable the ROCm build" |
| 127 | + exit 1 |
| 128 | +fi |
| 129 | +if ! grep -q 'find_dependency(hip CONFIG)' \ |
| 130 | + cmake-out-rocm-llm/executorch-backend-dependencies.cmake; then |
| 131 | + echo "ROCm package metadata does not declare its HIP dependency" |
| 132 | + exit 1 |
| 133 | +fi |
| 134 | +if grep -q 'find_dependency(CUDAToolkit)' \ |
| 135 | + cmake-out-rocm-llm/executorch-backend-dependencies.cmake; then |
| 136 | + echo "ROCm package metadata unexpectedly depends on CUDAToolkit" |
| 137 | + exit 1 |
| 138 | +fi |
| 139 | + |
| 140 | +export EXECUTORCH_EXECUTOR_RUNNER="${PWD}/cmake-out-rocm-llm/executor_runner" |
| 141 | +ROCM_EXCLUDED_TESTS=( |
| 142 | + # This focused job does not install the optional flash-linear-attention package. |
| 143 | + --ignore=backends/cuda/tests/test_chunk_gated_delta_rule.py |
| 144 | + # Defer specialized SDPA suites until their CI cost is measured. |
| 145 | + --ignore=backends/cuda/tests/test_tq4_sdpa.py |
| 146 | + --ignore=backends/cuda/tests/test_triton_sdpa.py |
| 147 | + --ignore=backends/cuda/tests/test_triton_sdpa_splitk.py |
| 148 | + # This test process core-dumps on ROCm instead of reporting a failure. |
| 149 | + --ignore=backends/cuda/tests/test_triton_sdpa_nan.py |
| 150 | +) |
| 151 | +python -m pytest -v -o 'addopts=' \ |
| 152 | + "${ROCM_EXCLUDED_TESTS[@]}" \ |
| 153 | + backends/cuda/tests \ |
| 154 | + backends/cuda/passes/tests |
| 155 | + |
| 156 | +ROCM_POINTWISE_DIR="${ROCM_CI_TMPDIR}/pointwise" |
| 157 | +python -m examples.cuda.scripts.export_amd_pointwise \ |
| 158 | + --output-dir "${ROCM_POINTWISE_DIR}" |
0 commit comments