Skip to content
Fly DSL test

Add common VectorType alias #2617

Sign in to view logs

Add common VectorType alias

Add common VectorType alias #2617

Workflow file for this run

.github/workflows/flydsl.yaml at 1468b26
name: Fly DSL test
on:
push:
branches:
- main
pull_request:
branches:
- main
types: [opened, synchronize, reopened, labeled]
workflow_dispatch:
permissions:
contents: read
actions: read
pull-requests: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
DOCKER_IMAGE: "rocm/pytorch:rocm7.2_ubuntu24.04_py3.12_pytorch_release_2.9.1"
GITHUB_REPO_NAME: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.event.head_commit.id || github.sha }}
jobs:
check-signal:
if: ${{ github.event_name != 'pull_request' || github.event.action != 'labeled' || github.event.label.name == 'multi-gpu' }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Wait for Checks workflow
run: ./.github/scripts/check_signal.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_SHA: ${{ github.sha }}
MAX_RETRIES: 5
RETRY_INTERVAL_SECONDS: 30
# ---------------------------------------------------------------------------
# Single-GPU tests: kernels, unit, examples, MLIR FileCheck, benchmarks.
# Runs on 1-GPU and Navi runners only.
# ---------------------------------------------------------------------------
test:
needs: check-signal
env:
# Some self-hosted runners configure a GitHub -> git-cache URL rewrite.
# If that cache is unavailable, checkout fails before the tests start.
GIT_CONFIG_GLOBAL: /dev/null
GIT_CONFIG_NOSYSTEM: "1"
strategy:
matrix:
runners: [
'linux-flydsl-mi325-1',
'linux-flydsl-mi355-1',
'linux-flydsl-navi-2',
]
fail-fast: false
runs-on: ${{ matrix.runners }}
steps:
- name: Checkout code
uses: actions/checkout@v4
env:
# Some self-hosted runners configure a GitHub -> git-cache URL rewrite.
# If that cache is unavailable, checkout fails before the tests start.
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/flydsl-gitconfig
GIT_CONFIG_NOSYSTEM: "1"
with:
repository: ${{ env.GITHUB_REPO_NAME }}
ref: ${{ env.GITHUB_COMMIT_SHA }}
path: flydsl-test
- name: Start CI container
run: |
echo "Clean up containers..."
docker ps -aq -f name=flydsl_test | xargs -r docker stop | xargs -r docker rm || true
echo "Start CI container..."
if [ -f "/etc/podinfo/gha-render-devices" ]; then
DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
else
DEVICE_FLAG="--device /dev/dri"
fi
echo "Starting container: flydsl_test:ci"
docker run -dt --network=host --user root --device=/dev/kfd $DEVICE_FLAG \
-v "${GITHUB_WORKSPACE:-$PWD}/flydsl-test:/flydsl-test" \
--ipc=host --group-add video \
--shm-size 16g \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
-w /flydsl-test \
--name flydsl_test \
${{ env.DOCKER_IMAGE }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
docker exec flydsl_test bash -c "apt-get update && apt-get install -y cmake build-essential patchelf"
docker exec flydsl_test bash -c "python3 -m pip install -U pip setuptools wheel"
docker exec flydsl_test bash -c "python3 -m pip install ninja>=1.11.1"
docker exec flydsl_test bash -c "python3 -m pip install -U 'hypothesis>=6.82.0'"
docker exec flydsl_test bash -c "git config --global --add safe.directory /flydsl-test && cd /flydsl-test && git log"
# Cache MLIR install tarball across workflow runs.
- name: Restore cached MLIR install tarball (if available)
id: mlir-cache
uses: actions/cache@v4
with:
path: mlir_install.tgz
# Key includes LLVM commit and hashes of build scripts/workflow.
# Repo is checked out under `flydsl-test/` (see actions/checkout path), so hash paths must include it.
key: mlir-install-${{ matrix.runners }}-${{ hashFiles('flydsl-test/thirdparty/llvm-hash.txt', 'flydsl-test/scripts/build_llvm.sh', 'flydsl-test/CMakeLists.txt', 'flydsl-test/.github/workflows/flydsl.yaml') }}
- name: Use cached MLIR install tarball (skip LLVM build)
if: steps.mlir-cache.outputs.cache-hit == 'true'
run: |
ls -lh mlir_install.tgz
docker cp mlir_install.tgz flydsl_test:/tmp/mlir_install.tgz
docker exec flydsl_test bash -c "rm -rf /llvm-project/mlir_install && mkdir -p /llvm-project && tar -xzf /tmp/mlir_install.tgz -C /llvm-project"
docker exec flydsl_test bash -c "ls -la /llvm-project/mlir_install/lib/cmake/mlir"
- name: Build LLVM
if: steps.mlir-cache.outputs.cache-hit != 'true'
run: |
set -ex
docker exec flydsl_test bash -c "cd /flydsl-test && bash scripts/build_llvm.sh"
docker exec flydsl_test bash -c "ls -la /llvm-project/mlir_install/lib/cmake/mlir"
docker cp flydsl_test:/llvm-project/mlir_install.tgz ./mlir_install.tgz || true
ls -lh ./mlir_install.tgz || true
- name: Build FlyDSL (uses MLIR install prefix)
run: |
docker exec flydsl_test bash -c "export MLIR_PATH=/llvm-project/mlir_install && cd /flydsl-test && python3 -m pip install -e . --use-pep517"
- name: MLIR tarball artifact status
run: |
echo "cache-hit: ${{ steps.mlir-cache.outputs.cache-hit }}"
echo "event: ${{ github.event_name }}"
echo "ref: ${{ github.ref }}"
if [[ "${{ steps.mlir-cache.outputs.cache-hit }}" == "true" ]]; then
echo "artifact: skipped (MLIR restored from cache; no rebuild)"
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
echo "artifact: will upload (workflow_dispatch + rebuilt)"
elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then
echo "artifact: will upload (push to main + rebuilt, e.g. PR merged)"
else
echo "artifact: skipped (PR runs do not upload)"
fi
- name: Export MLIR install tarball (workflow_dispatch OR push-to-main)
if: steps.mlir-cache.outputs.cache-hit != 'true' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main'))
run: |
ls -lh ./mlir_install.tgz || true
- name: Upload MLIR install tarball artifact (workflow_dispatch OR push-to-main)
if: steps.mlir-cache.outputs.cache-hit != 'true' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main'))
uses: actions/upload-artifact@v4
with:
name: mlir_install-${{ matrix.runners }}-${{ hashFiles('flydsl-test/thirdparty/llvm-hash.txt') }}
path: mlir_install.tgz
if-no-files-found: ignore
retention-days: 7
- name: Prepare aiter
if: ${{ !contains(matrix.runners, 'navi') }}
run: |
docker exec flydsl_test bash -c "rm -rf /tmp/aiter && git clone --depth 1 --recursive --shallow-submodules https://github.com/ROCm/aiter.git /tmp/aiter"
docker exec flydsl_test bash -c "python3 -c \"from pathlib import Path; src = Path('/tmp/aiter/requirements.txt'); dst = Path('/tmp/aiter/requirements-flydsl-ci.txt'); lines = [line for line in src.read_text().splitlines() if line.strip() and not line.strip().startswith('flydsl==')]; dst.write_text('\\n'.join(lines) + '\\n')\" && python3 -m pip install -r /tmp/aiter/requirements-flydsl-ci.txt"
docker exec flydsl_test bash -c "python3 -m pip uninstall -y triton pytorch-triton pytorch-triton-rocm triton-rocm amd-triton || true"
docker exec flydsl_test bash -c "python3 -m pip install --extra-index-url https://pypi.amd.com/triton/rocm-7.2.0/simple/ triton"
docker exec flydsl_test bash -c "python3 -c 'import triton; assert tuple(map(int, triton.__version__.split(\"+\", 1)[0].split(\".\")[:3])) >= (3, 6, 0), triton.__version__; print(\"Installed triton\", triton.__version__)'"
- name: Run tests
run: |
docker exec flydsl_test bash -c "export PYTHONPATH=/tmp/aiter:\${PYTHONPATH:-} && export AITER_REPO=/tmp/aiter && cd /flydsl-test && RUN_TESTS_FULL=1 bash scripts/run_tests.sh"
- name: Show tests logs
if: failure()
run: |
docker exec flydsl_test bash -c 'cd /tmp && tar czf /tmp/logs.tgz *.log 2>/dev/null || echo "no logs"'
docker cp flydsl_test:/tmp/logs.tgz . || true
if [ -f logs.tgz ]; then
tar -xzf logs.tgz || true
cat *.log || true
else
echo "logs.tgz not found; skipping log extraction"
fi
- name: Run benchmarks
id: benchmarks
run: |
docker exec -i flydsl_test bash <<'BASH'
set -e -o pipefail
export PYTHONPATH=/tmp/aiter:${PYTHONPATH:-}
export AITER_REPO=/tmp/aiter
cd /flydsl-test
BENCH_LOG_DIR=/tmp/flydsl_bench_current bash scripts/run_benchmark.sh 2>&1 | tee /tmp/bench_current.out
python3 scripts/benchmark_output_to_csv.py /tmp/bench_current.out /tmp/bench_current.csv
BASH
- name: Run benchmark baselines
id: bench-baselines
timeout-minutes: 180
continue-on-error: true
run: |
docker exec -i flydsl_test bash <<'BASH'
set -u
cd /flydsl-test
rm -rf /tmp/flydsl-bench-main-* /tmp/flydsl-bench-tag
rm -f /tmp/bench_main.csv /tmp/bench_latest_tag.csv /tmp/bench_main_label /tmp/bench_latest_tag_label
git worktree prune || true
git fetch origin main --depth=8
found_main=0
idx=0
for commit in $(git rev-list --max-count=8 FETCH_HEAD); do
label="main"
if [ "${idx}" -gt 0 ]; then
label="main~${idx}"
fi
worktree="/tmp/flydsl-bench-main-${idx}"
csv="/tmp/bench_candidate_${idx}.csv"
output="/tmp/bench_candidate_${idx}.out"
log_dir="/tmp/flydsl_bench_${label}"
echo "Trying benchmark baseline ${label} (${commit})"
if ! git worktree add --detach "${worktree}" "${commit}"; then
echo "Failed to create worktree for ${label}; trying older main commit."
idx=$((idx + 1))
continue
fi
(
set -e -o pipefail
cd "${worktree}"
export MLIR_PATH=/llvm-project/mlir_install
python3 -m pip install -e . --use-pep517 2>&1 | tail -5
export PYTHONPATH=/tmp/aiter:${PYTHONPATH:-}
export AITER_REPO=/tmp/aiter
BENCH_LOG_DIR="${log_dir}" bash scripts/run_benchmark.sh 2>&1 | tee "${output}"
python3 /flydsl-test/scripts/benchmark_output_to_csv.py "${output}" "${csv}"
)
status=$?
if [ "${status}" -eq 0 ] && [ -s "${csv}" ]; then
cp "${csv}" /tmp/bench_main.csv
echo "${label}" >/tmp/bench_main_label
found_main=1
break
else
echo "Benchmark baseline ${label} failed; trying older main commit."
fi
idx=$((idx + 1))
done
if [ "${found_main}" -eq 0 ]; then
echo "No usable main benchmark baseline found."
fi
if git fetch origin 'refs/tags/v*:refs/tags/v*' --force --depth=1; then
latest_tag="$(git tag --list 'v*' --sort=-v:refname | sed -n '1p')"
if [ -n "${latest_tag}" ]; then
worktree="/tmp/flydsl-bench-tag"
csv="/tmp/bench_latest_tag_candidate.csv"
output="/tmp/bench_latest_tag.out"
log_dir="/tmp/flydsl_bench_${latest_tag}"
package_version="${latest_tag#v}"
echo "Trying benchmark baseline ${latest_tag} from pip package flydsl==${package_version}"
if git worktree add --detach "${worktree}" "${latest_tag}"; then
(
set -e -o pipefail
cd "${worktree}"
python3 -m pip install --only-binary=:all: "flydsl==${package_version}" 2>&1 | tail -5
export PYTHONPATH=/tmp/aiter:${PYTHONPATH:-}
export AITER_REPO=/tmp/aiter
BENCH_LOG_DIR="${log_dir}" bash scripts/run_benchmark.sh 2>&1 | tee "${output}"
python3 /flydsl-test/scripts/benchmark_output_to_csv.py "${output}" "${csv}"
)
status=$?
if [ "${status}" -eq 0 ] && [ -s "${csv}" ]; then
cp "${csv}" /tmp/bench_latest_tag.csv
echo "${latest_tag}" >/tmp/bench_latest_tag_label
else
echo "Benchmark baseline ${latest_tag} failed."
fi
else
echo "Failed to create worktree for ${latest_tag}; skipping latest-tag comparison."
fi
else
echo "No v* tags found; skipping latest-tag comparison."
fi
else
echo "Failed to fetch tags; skipping latest-tag comparison."
fi
BASH
- name: Check benchmark performance (current vs main)
if: steps.bench-baselines.outcome != 'skipped'
timeout-minutes: 5
run: |
docker exec flydsl_test bash -c "
if [ ! -f /tmp/bench_main.csv ]; then
echo 'No usable main benchmark baseline found; skipping main comparison.'
exit 0
fi
cd /flydsl-test
main_label=\$(cat /tmp/bench_main_label 2>/dev/null || echo main)
python3 scripts/compare_benchmark.py /tmp/bench_main.csv /tmp/bench_current.csv \
--baseline-label \"\${main_label}\" --current-label current
"
- name: Check benchmark performance (current vs latest tag)
if: steps.bench-baselines.outcome != 'skipped'
timeout-minutes: 5
run: |
docker exec flydsl_test bash -c "
if [ ! -f /tmp/bench_latest_tag.csv ]; then
echo 'No usable latest-tag benchmark baseline found; skipping latest-tag comparison.'
exit 0
fi
cd /flydsl-test
latest_tag_label=\$(cat /tmp/bench_latest_tag_label 2>/dev/null || echo latest-tag)
python3 scripts/compare_benchmark.py /tmp/bench_latest_tag.csv /tmp/bench_current.csv \
--baseline-label \"\${latest_tag_label}\" --current-label current
"
- name: Show benchmarks logs
if: failure()
run: |
docker exec flydsl_test bash -c 'cd /tmp && tar czf /tmp/flydsl_bench_logs.tgz flydsl_bench* bench_*.csv 2>/dev/null || echo "no logs"'
docker cp flydsl_test:/tmp/flydsl_bench_logs.tgz . || true
if [ -f flydsl_bench_logs.tgz ]; then
tar -xzf flydsl_bench_logs.tgz || true
cat flydsl_bench*/*.log || true
cat bench_*.csv || true
else
echo "flydsl_bench_logs.tgz not found; skipping log extraction"
fi
- name: Clean up
if: always()
run: |
docker stop flydsl_test
docker rm flydsl_test
# ---------------------------------------------------------------------------
# Multi-GPU communication operator tests: ONLY for 8-GPU runners.
# Runs on BOTH linux-flydsl-mi325-8 AND linux-flydsl-mi355-8 independently.
# Triggered when PR has label "multi-gpu" (added by a maintainer), or when
# the workflow is manually dispatched.
# fail-fast: false ensures both runners always complete even if one fails.
# ---------------------------------------------------------------------------
multi-gpu:
needs: test
name: Multi-GPU Communication Operator Tests (${{ matrix.runners }})
timeout-minutes: 120
env:
# Keep checkout independent of runner-local git-cache rewrites.
GIT_CONFIG_GLOBAL: /dev/null
GIT_CONFIG_NOSYSTEM: "1"
if: |
always() && (
(github.event_name == 'pull_request' &&
contains(github.event.pull_request.labels.*.name, 'multi-gpu')) ||
github.event_name == 'workflow_dispatch'
)
strategy:
matrix:
runners: [
'linux-flydsl-mi325-8',
'linux-flydsl-mi355-8',
]
fail-fast: false
runs-on: ${{ matrix.runners }}
steps:
- name: Checkout code
uses: actions/checkout@v4
env:
# Keep checkout independent of runner-local git-cache rewrites.
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/flydsl-gitconfig
GIT_CONFIG_NOSYSTEM: "1"
with:
repository: ${{ env.GITHUB_REPO_NAME }}
ref: ${{ env.GITHUB_COMMIT_SHA }}
path: flydsl-test
- name: Start CI container
run: |
echo "Clean up containers..."
docker ps -aq -f name=flydsl_test | xargs -r docker stop | xargs -r docker rm || true
echo "Start CI container..."
if [ -f "/etc/podinfo/gha-render-devices" ]; then
DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
else
DEVICE_FLAG="--device /dev/dri"
fi
docker run -dt --network=host --user root --device=/dev/kfd $DEVICE_FLAG \
-v "${GITHUB_WORKSPACE:-$PWD}/flydsl-test:/flydsl-test" \
--ipc=host --group-add video \
--shm-size 16g \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
-w /flydsl-test \
--name flydsl_test \
${{ env.DOCKER_IMAGE }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
docker exec flydsl_test bash -c "apt-get update && apt-get install -y cmake build-essential patchelf"
docker exec flydsl_test bash -c "python3 -m pip install -U pip setuptools wheel"
docker exec flydsl_test bash -c "python3 -m pip install ninja>=1.11.1"
docker exec flydsl_test bash -c "python3 -m pip install -U 'hypothesis>=6.82.0'"
docker exec flydsl_test bash -c "git config --global --add safe.directory /flydsl-test && cd /flydsl-test && git log"
- name: Restore cached MLIR install tarball (if available)
id: mlir-cache
uses: actions/cache@v4
with:
path: mlir_install.tgz
key: mlir-install-${{ matrix.runners }}-${{ hashFiles('flydsl-test/thirdparty/llvm-hash.txt', 'flydsl-test/scripts/build_llvm.sh', 'flydsl-test/CMakeLists.txt', 'flydsl-test/.github/workflows/flydsl.yaml') }}
- name: Use cached MLIR install tarball (skip LLVM build)
if: steps.mlir-cache.outputs.cache-hit == 'true'
run: |
ls -lh mlir_install.tgz
docker cp mlir_install.tgz flydsl_test:/tmp/mlir_install.tgz
docker exec flydsl_test bash -c "rm -rf /llvm-project/mlir_install && mkdir -p /llvm-project && tar -xzf /tmp/mlir_install.tgz -C /llvm-project"
docker exec flydsl_test bash -c "ls -la /llvm-project/mlir_install/lib/cmake/mlir"
- name: Build LLVM
if: steps.mlir-cache.outputs.cache-hit != 'true'
run: |
set -ex
docker exec flydsl_test bash -c "cd /flydsl-test && bash scripts/build_llvm.sh"
docker exec flydsl_test bash -c "ls -la /llvm-project/mlir_install/lib/cmake/mlir"
docker cp flydsl_test:/llvm-project/mlir_install.tgz ./mlir_install.tgz || true
- name: Build FlyDSL (uses MLIR install prefix)
run: |
docker exec flydsl_test bash -c "export MLIR_PATH=/llvm-project/mlir_install && cd /flydsl-test && python3 -m pip install -e . --use-pep517"
- name: Install mori (for shmem regression test)
timeout-minutes: 15
run: |
docker exec flydsl_test bash -c "
apt-get install -y libpci-dev libibverbs-dev libgrpc++1.51 libgrpc29 &&
python3 -m pip install amd_mori &&
MORI_PRECOMPILE=1 python3 -c 'import mori'
"
- name: Run multi-GPU shmem regression test
timeout-minutes: 10
run: |
docker exec flydsl_test bash -c "
cd /flydsl-test
python3 -m pytest tests/kernels/test_flydsl_shmem.py \
-m multi_gpu -v --no-header --tb=short
"
- name: Run multi-GPU dispatch/combine CI sweep (8-GPU, accuracy + cudagraph perf)
timeout-minutes: 30
run: |
docker exec flydsl_test bash -c "
cd /flydsl-test &&
python tests/kernels/test_profiler_dispatch_combine.py \
--ci-sweep \
--world-size 8 \
--port 29503 \
--output-dir /tmp/flydsl_ci_sweep
"
- name: Run multi-GPU allreduce tests
timeout-minutes: 30
run: |
docker exec flydsl_test bash -c "
cd /flydsl-test
python3 -m pytest tests/kernels/test_allreduce.py \
-m multi_gpu -v --no-header --tb=short
"
- name: Run allreduce benchmark (PR)
timeout-minutes: 30
run: |
docker exec flydsl_test bash -c "
cd /flydsl-test
python3 tests/kernels/test_allreduce.py \
--world_size 8 --iters 51 --warmup 5 \
--allreduce_impl flydsl --mode cudagraph \
--shapes '2,7168,fp16;32,8192,fp32;128,8192,fp16;1024,7168,bf16;4096,8192,bf16' \
--output_csv /tmp/bench_pr.csv
"
- name: Build main branch baseline
id: build-main
timeout-minutes: 20
continue-on-error: true
run: |
docker exec flydsl_test bash -c "
cd /flydsl-test
git fetch origin main --depth=1
git worktree add /tmp/flydsl-main origin/main
cd /tmp/flydsl-main
export MLIR_PATH=/llvm-project/mlir_install
python3 -m pip install -e . --use-pep517 2>&1 | tail -5
"
- name: Run allreduce benchmark (main)
id: bench-main
if: steps.build-main.outcome == 'success'
timeout-minutes: 30
continue-on-error: true
run: |
docker exec flydsl_test bash -c "
cp /flydsl-test/tests/kernels/test_allreduce.py \
/tmp/flydsl-main/tests/kernels/test_allreduce.py
cd /tmp/flydsl-main
python3 tests/kernels/test_allreduce.py \
--world_size 8 --iters 51 --warmup 5 \
--allreduce_impl flydsl --mode cudagraph \
--shapes '2,7168,fp16;32,8192,fp32;128,8192,fp16;1024,7168,bf16;4096,8192,bf16' \
--output_csv /tmp/bench_main.csv
"
- name: Check performance regression (PR vs main)
if: steps.bench-main.outcome != 'skipped'
timeout-minutes: 5
run: |
docker exec flydsl_test bash -c "
cd /flydsl-test
python3 tests/kernels/compare_allreduce_benchmark.py \
/tmp/bench_main.csv /tmp/bench_pr.csv
"
- name: Show test logs
if: failure()
run: |
docker exec flydsl_test bash -c 'cd /tmp && tar czf /tmp/logs.tgz *.log 2>/dev/null || echo "no logs"'
docker cp flydsl_test:/tmp/logs.tgz . || true
if [ -f logs.tgz ]; then
tar -xzf logs.tgz || true
cat *.log || true
else
echo "logs.tgz not found; skipping log extraction"
fi
- name: Clean up
if: always()
run: |
docker stop flydsl_test
docker rm flydsl_test