FlyDSL ATOM nightly #5
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: FlyDSL ATOM nightly | |
| on: | |
| schedule: | |
| # Run after the FlyDSL nightly wheel publish/promote flow. | |
| - cron: '0 4 * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| ATOM_BASE_IMAGE: rocm/atom-dev:latest | |
| ATOM_REPOSITORY: ROCm/ATOM | |
| ATOM_REF: main | |
| FLYDSL_WHEEL_BUCKET: framework-whls-nightlies | |
| FLYDSL_WHEEL_PREFIX: whl/gfx942-gfx950 | |
| jobs: | |
| atom-accuracy: | |
| name: ATOM accuracy / ${{ matrix.model_name }} | |
| timeout-minutes: 180 | |
| runs-on: ${{ matrix.runner }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - model_name: DeepSeek-R1-0528 | |
| model_path: deepseek-ai/DeepSeek-R1-0528 | |
| extra_args: --kv_cache_dtype fp8 -tp 8 | |
| accuracy_threshold: 0.94 | |
| runner: linux-flydsl-mi355-8 | |
| env_vars: "" | |
| - model_name: gpt-oss-120b | |
| model_path: openai/gpt-oss-120b | |
| extra_args: --kv_cache_dtype fp8 --gpu-memory-utilization 0.3 | |
| accuracy_threshold: 0.38 | |
| runner: linux-flydsl-mi355-1 | |
| env_vars: "" | |
| env: | |
| CONTAINER_NAME: flydsl_atom_${{ strategy.job-index }} | |
| steps: | |
| - name: Checkout FlyDSL utility scripts | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: | | |
| scripts/install_awscli.sh | |
| path: flydsl | |
| - name: Checkout upstream ATOM repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.ATOM_REPOSITORY }} | |
| ref: ${{ env.ATOM_REF }} | |
| path: atom-upstream | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::661452401056:role/framework-flydsl-nightlies | |
| - name: Install AWS CLI | |
| run: bash ./flydsl/scripts/install_awscli.sh | |
| - name: Start CI container | |
| run: | | |
| set -euo pipefail | |
| echo "Cleaning up any stale container..." | |
| (docker ps -aq -f name="^${CONTAINER_NAME}$" | xargs -r docker stop) || true | |
| (docker ps -aq -f name="^${CONTAINER_NAME}$" | xargs -r docker rm) || true | |
| if [ -f "/etc/podinfo/gha-render-devices" ]; then | |
| DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices) | |
| else | |
| DEVICE_FLAG="--device /dev/dri" | |
| fi | |
| MODEL_MOUNT="" | |
| if [ -d "/models" ]; then | |
| MODEL_MOUNT="-v /models:/models" | |
| fi | |
| printf '%s\n' "${MODEL_ENV_VARS:-}" | grep -v '^$' > /tmp/flydsl_atom_env.txt || true | |
| docker run -dt --pull always --device=/dev/kfd $DEVICE_FLAG \ | |
| -v "${GITHUB_WORKSPACE:-$PWD}:/workspace" \ | |
| $MODEL_MOUNT \ | |
| -w /workspace \ | |
| --ipc=host --group-add video \ | |
| --shm-size=16G \ | |
| --privileged \ | |
| --cap-add=SYS_PTRACE \ | |
| --env-file /tmp/flydsl_atom_env.txt \ | |
| --security-opt seccomp=unconfined \ | |
| --ulimit memlock=-1 \ | |
| --ulimit stack=67108864 \ | |
| -e ATOM_DISABLE_MMAP=true \ | |
| -e HF_HUB_ENABLE_HF_TRANSFER=1 \ | |
| -e HF_TOKEN="${HF_TOKEN:-${AMD_HF_TOKEN:-${HF_TOKEN_TEST:-}}}" \ | |
| --name "${CONTAINER_NAME}" \ | |
| "${{ env.ATOM_BASE_IMAGE }}" | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| MODEL_ENV_VARS: ${{ matrix.env_vars }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| AMD_HF_TOKEN: ${{ secrets.AMD_HF_TOKEN }} | |
| HF_TOKEN_TEST: ${{ secrets.HF_TOKEN_TEST }} | |
| - name: Install CI dependencies | |
| run: | | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| command -v git curl python3 | |
| git --version | |
| python3 --version | |
| python3 -m pip install --timeout 60 --retries 10 -U pip setuptools wheel | |
| python3 -m pip install --timeout 60 --retries 10 'lm-eval[api]' hf_transfer 'huggingface_hub[cli]' | |
| chmod +x /workspace/atom-upstream/.github/scripts/atom_test.sh | |
| " | |
| - name: Download latest FlyDSL nightly wheel | |
| run: | | |
| set -euo pipefail | |
| PY_TAG=$(docker exec "${CONTAINER_NAME}" python3 - <<'PY' | |
| import sys | |
| print(f"cp{sys.version_info.major}{sys.version_info.minor}") | |
| PY | |
| ) | |
| echo "Detected Python tag inside atom-dev image: ${PY_TAG}" | |
| aws s3api list-objects-v2 \ | |
| --bucket "${{ env.FLYDSL_WHEEL_BUCKET }}" \ | |
| --prefix "${{ env.FLYDSL_WHEEL_PREFIX }}/" \ | |
| --output json > /tmp/flydsl-nightly-wheels.json | |
| LATEST_KEY=$(PY_TAG="${PY_TAG}" S3_PREFIX="${{ env.FLYDSL_WHEEL_PREFIX }}" python3 - <<'PY' | |
| import datetime as dt | |
| import json | |
| import os | |
| import posixpath | |
| py_tag = os.environ["PY_TAG"] | |
| s3_prefix = os.environ["S3_PREFIX"] | |
| with open("/tmp/flydsl-nightly-wheels.json", encoding="utf-8") as f: | |
| data = json.load(f) | |
| candidates = [] | |
| for obj in data.get("Contents", []): | |
| key = obj["Key"] | |
| name = posixpath.basename(key) | |
| if not (name.startswith("flydsl-") and name.endswith(".whl")): | |
| continue | |
| if py_tag not in name: | |
| continue | |
| timestamp = dt.datetime.fromisoformat(obj["LastModified"].replace("Z", "+00:00")) | |
| candidates.append((timestamp, key)) | |
| if not candidates: | |
| raise SystemExit(f"No FlyDSL wheel found for {py_tag} under s3://framework-whls-nightlies/{s3_prefix}/") | |
| print(max(candidates, key=lambda item: item[0])[1]) | |
| PY | |
| ) | |
| WHEEL_NAME=$(basename "${LATEST_KEY}") | |
| WHEEL_S3_URI="s3://${{ env.FLYDSL_WHEEL_BUCKET }}/${LATEST_KEY}" | |
| echo "Selected FlyDSL wheel: ${WHEEL_S3_URI}" | |
| aws s3 cp "${WHEEL_S3_URI}" "./${WHEEL_NAME}" | |
| printf 'Wheel S3 URI: %s\n' "${WHEEL_S3_URI}" | tee selected-flydsl-wheel.txt | |
| echo "FLYDSL_WHEEL_NAME=${WHEEL_NAME}" >> "${GITHUB_ENV}" | |
| echo "FLYDSL_WHEEL_S3_URI=${WHEEL_S3_URI}" >> "${GITHUB_ENV}" | |
| - name: Install latest FlyDSL nightly wheel | |
| run: | | |
| set -euo pipefail | |
| docker cp "${FLYDSL_WHEEL_NAME}" "${CONTAINER_NAME}:/tmp/${FLYDSL_WHEEL_NAME}" | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| pip uninstall -y flydsl || true | |
| python3 -m pip install --force-reinstall /tmp/${FLYDSL_WHEEL_NAME} | |
| python3 - <<'PY' | |
| import flydsl | |
| print(f'flydsl package: {flydsl.__file__}') | |
| print(f'flydsl version: {getattr(flydsl, "__version__", "unknown")}') | |
| PY | |
| " | |
| - name: Record nightly wheel | |
| run: | | |
| { | |
| echo "## Nightly Wheel" | |
| echo | |
| echo "- S3 URI: \`${FLYDSL_WHEEL_S3_URI}\`" | |
| echo "- Local file: \`${FLYDSL_WHEEL_NAME}\`" | |
| } >> "${GITHUB_STEP_SUMMARY}" | |
| - name: Pre-download model when cache mount exists | |
| run: | | |
| set -euo pipefail | |
| if [ -d "/models" ]; then | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| hf download '${{ matrix.model_path }}' --local-dir '/models/${{ matrix.model_path }}' | |
| " | |
| else | |
| echo "/models is not available on this runner; using Hugging Face model path directly." | |
| fi | |
| - name: Run ATOM accuracy test | |
| run: | | |
| set -euo pipefail | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| cd /workspace/atom-upstream | |
| if [ -d '/models/${{ matrix.model_path }}' ]; then | |
| model_path='/models/${{ matrix.model_path }}' | |
| else | |
| model_path='${{ matrix.model_path }}' | |
| fi | |
| ATOM_SERVER_LOG=/tmp/atom_server.log \ | |
| ATOM_CLIENT_LOG=/tmp/atom_client.log \ | |
| ATOM_DOCKER_IMAGE='${{ env.ATOM_BASE_IMAGE }}' \ | |
| bash .github/scripts/atom_test.sh launch \"\$model_path\" ${{ matrix.extra_args }} | |
| ATOM_SERVER_LOG=/tmp/atom_server.log \ | |
| ATOM_CLIENT_LOG=/tmp/atom_client.log \ | |
| ATOM_DOCKER_IMAGE='${{ env.ATOM_BASE_IMAGE }}' \ | |
| bash .github/scripts/atom_test.sh accuracy \"\$model_path\" | |
| " 2>&1 | tee atom_accuracy_output.txt | |
| - name: Check accuracy threshold | |
| if: success() | |
| env: | |
| ACCURACY_THRESHOLD: ${{ matrix.accuracy_threshold }} | |
| run: | | |
| set -euo pipefail | |
| RESULT_FILE=$(python3 - <<'PY' | |
| import glob | |
| files = sorted(glob.glob('atom-upstream/accuracy_test_results/*.json')) | |
| print(files[-1] if files else '') | |
| PY | |
| ) | |
| if [ -z "${RESULT_FILE:-}" ] || [ ! -f "${RESULT_FILE}" ]; then | |
| echo "No accuracy result JSON found in atom-upstream/accuracy_test_results/" | |
| exit 1 | |
| fi | |
| RESULT_FILE="${RESULT_FILE}" python3 - <<'PY' | |
| import json | |
| import os | |
| result_file = os.environ["RESULT_FILE"] | |
| threshold = float(os.environ["ACCURACY_THRESHOLD"]) | |
| with open(result_file, encoding="utf-8") as f: | |
| result = json.load(f) | |
| value = result["results"]["gsm8k"]["exact_match,flexible-extract"] | |
| print(f"RESULT_FILE: {result_file}") | |
| print(f"Flexible extract value: {value}") | |
| print(f"Accuracy threshold: {threshold}") | |
| if value < threshold: | |
| raise SystemExit(f"Accuracy test failed: {value} < {threshold}") | |
| print(f"Accuracy test passed: {value} >= {threshold}") | |
| PY | |
| - name: Collect test summary | |
| if: success() | |
| run: | | |
| echo "Accuracy Test Summary for ${{ matrix.model_name }}:" >> "${GITHUB_STEP_SUMMARY}" | |
| awk '/\|Tasks\|Version\|/,/^$/ { if (NF > 0) print }' atom_accuracy_output.txt >> "${GITHUB_STEP_SUMMARY}" || true | |
| - name: Collect ATOM logs | |
| if: always() | |
| run: | | |
| docker cp "${CONTAINER_NAME}:/tmp/atom_server.log" atom_server.log 2>/dev/null || true | |
| docker cp "${CONTAINER_NAME}:/tmp/atom_client.log" atom_client.log 2>/dev/null || true | |
| - name: Upload logs and accuracy results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: flydsl-atom-${{ matrix.model_name }} | |
| path: | | |
| selected-flydsl-wheel.txt | |
| atom_accuracy_output.txt | |
| atom_server.log | |
| atom_client.log | |
| atom-upstream/accuracy_test_results/*.json | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| cd /workspace/atom-upstream | |
| bash .github/scripts/atom_test.sh stop || true | |
| " || true | |
| docker stop "${CONTAINER_NAME}" || true | |
| docker rm "${CONTAINER_NAME}" || true |