FlyDSL ATOM nightly #6
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: FlyDSL ATOM nightly | |
| on: | |
| schedule: | |
| # Run after the FlyDSL nightly wheel publish/promote flow. | |
| - cron: '0 4 * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| ATOM_BASE_IMAGE: rocm/atom-dev:latest | |
| ATOM_REPOSITORY: ROCm/ATOM | |
| ATOM_REF: main | |
| jobs: | |
| atom-accuracy: | |
| name: ATOM accuracy / ${{ matrix.model_name }} | |
| timeout-minutes: 180 | |
| runs-on: ${{ matrix.runner }} | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - model_name: DeepSeek-R1-0528 | |
| model_path: deepseek-ai/DeepSeek-R1-0528 | |
| extra_args: --kv_cache_dtype fp8 -tp 8 | |
| accuracy_threshold: 0.94 | |
| runner: linux-flydsl-mi355-8 | |
| env_vars: "" | |
| - model_name: gpt-oss-120b | |
| model_path: openai/gpt-oss-120b | |
| extra_args: --kv_cache_dtype fp8 --gpu-memory-utilization 0.3 | |
| accuracy_threshold: 0.38 | |
| runner: linux-flydsl-mi355-1 | |
| env_vars: "" | |
| env: | |
| CONTAINER_NAME: flydsl_atom_${{ strategy.job-index }} | |
| steps: | |
| - name: Checkout upstream ATOM repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.ATOM_REPOSITORY }} | |
| ref: ${{ env.ATOM_REF }} | |
| path: atom-upstream | |
| - name: Start CI container | |
| run: | | |
| set -euo pipefail | |
| echo "Cleaning up any stale container..." | |
| (docker ps -aq -f name="^${CONTAINER_NAME}$" | xargs -r docker stop) || true | |
| (docker ps -aq -f name="^${CONTAINER_NAME}$" | xargs -r docker rm) || true | |
| if [ -f "/etc/podinfo/gha-render-devices" ]; then | |
| DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices) | |
| else | |
| DEVICE_FLAG="--device /dev/dri" | |
| fi | |
| MODEL_MOUNT="" | |
| if [ -d "/models" ]; then | |
| MODEL_MOUNT="-v /models:/models" | |
| fi | |
| printf '%s\n' "${MODEL_ENV_VARS:-}" | grep -v '^$' > /tmp/flydsl_atom_env.txt || true | |
| docker run -dt --pull always --device=/dev/kfd $DEVICE_FLAG \ | |
| -v "${GITHUB_WORKSPACE:-$PWD}:/workspace" \ | |
| $MODEL_MOUNT \ | |
| -w /workspace \ | |
| --ipc=host --group-add video \ | |
| --shm-size=16G \ | |
| --privileged \ | |
| --cap-add=SYS_PTRACE \ | |
| --env-file /tmp/flydsl_atom_env.txt \ | |
| --security-opt seccomp=unconfined \ | |
| --ulimit memlock=-1 \ | |
| --ulimit stack=67108864 \ | |
| -e ATOM_DISABLE_MMAP=true \ | |
| -e HF_HUB_ENABLE_HF_TRANSFER=1 \ | |
| -e HF_TOKEN="${HF_TOKEN:-${AMD_HF_TOKEN:-${HF_TOKEN_TEST:-}}}" \ | |
| --name "${CONTAINER_NAME}" \ | |
| "${{ env.ATOM_BASE_IMAGE }}" | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| MODEL_ENV_VARS: ${{ matrix.env_vars }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| AMD_HF_TOKEN: ${{ secrets.AMD_HF_TOKEN }} | |
| HF_TOKEN_TEST: ${{ secrets.HF_TOKEN_TEST }} | |
| - name: Install CI dependencies | |
| run: | | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| command -v curl python3 | |
| python3 --version | |
| python3 -m pip install --timeout 60 --retries 10 -U pip setuptools wheel | |
| python3 -m pip install --timeout 60 --retries 10 'lm-eval[api]' hf_transfer 'huggingface_hub[cli]' | |
| chmod +x /workspace/atom-upstream/.github/scripts/atom_test.sh | |
| " | |
| - name: Check bundled FlyDSL | |
| run: | | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| python3 -m pip show flydsl || true | |
| python3 - <<'PY' | |
| import flydsl | |
| print(f'flydsl package: {flydsl.__file__}') | |
| print(f'flydsl version: {getattr(flydsl, "__version__", "unknown")}') | |
| PY | |
| " | |
| - name: Record bundled FlyDSL | |
| run: | | |
| { | |
| echo "## Bundled FlyDSL" | |
| echo | |
| echo "- Source: \`rocm/atom-dev:latest\`" | |
| echo "- Validation mode: use FlyDSL preinstalled in the nightly image" | |
| } >> "${GITHUB_STEP_SUMMARY}" | |
| - name: Pre-download model when cache mount exists | |
| run: | | |
| set -euo pipefail | |
| if [ -d "/models" ]; then | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| hf download '${{ matrix.model_path }}' --local-dir '/models/${{ matrix.model_path }}' | |
| " | |
| else | |
| echo "/models is not available on this runner; using Hugging Face model path directly." | |
| fi | |
| - name: Run ATOM accuracy test | |
| run: | | |
| set -euo pipefail | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| cd /workspace/atom-upstream | |
| if [ -d '/models/${{ matrix.model_path }}' ]; then | |
| model_path='/models/${{ matrix.model_path }}' | |
| else | |
| model_path='${{ matrix.model_path }}' | |
| fi | |
| ATOM_SERVER_LOG=/tmp/atom_server.log \ | |
| ATOM_CLIENT_LOG=/tmp/atom_client.log \ | |
| ATOM_DOCKER_IMAGE='${{ env.ATOM_BASE_IMAGE }}' \ | |
| bash .github/scripts/atom_test.sh launch \"\$model_path\" ${{ matrix.extra_args }} | |
| ATOM_SERVER_LOG=/tmp/atom_server.log \ | |
| ATOM_CLIENT_LOG=/tmp/atom_client.log \ | |
| ATOM_DOCKER_IMAGE='${{ env.ATOM_BASE_IMAGE }}' \ | |
| bash .github/scripts/atom_test.sh accuracy \"\$model_path\" | |
| " 2>&1 | tee atom_accuracy_output.txt | |
| - name: Check accuracy threshold | |
| if: success() | |
| env: | |
| ACCURACY_THRESHOLD: ${{ matrix.accuracy_threshold }} | |
| run: | | |
| set -euo pipefail | |
| RESULT_FILE=$(python3 - <<'PY' | |
| import glob | |
| files = sorted(glob.glob('atom-upstream/accuracy_test_results/*.json')) | |
| print(files[-1] if files else '') | |
| PY | |
| ) | |
| if [ -z "${RESULT_FILE:-}" ] || [ ! -f "${RESULT_FILE}" ]; then | |
| echo "No accuracy result JSON found in atom-upstream/accuracy_test_results/" | |
| exit 1 | |
| fi | |
| RESULT_FILE="${RESULT_FILE}" python3 - <<'PY' | |
| import json | |
| import os | |
| result_file = os.environ["RESULT_FILE"] | |
| threshold = float(os.environ["ACCURACY_THRESHOLD"]) | |
| with open(result_file, encoding="utf-8") as f: | |
| result = json.load(f) | |
| value = result["results"]["gsm8k"]["exact_match,flexible-extract"] | |
| print(f"RESULT_FILE: {result_file}") | |
| print(f"Flexible extract value: {value}") | |
| print(f"Accuracy threshold: {threshold}") | |
| if value < threshold: | |
| raise SystemExit(f"Accuracy test failed: {value} < {threshold}") | |
| print(f"Accuracy test passed: {value} >= {threshold}") | |
| PY | |
| - name: Collect test summary | |
| if: success() | |
| run: | | |
| echo "Accuracy Test Summary for ${{ matrix.model_name }}:" >> "${GITHUB_STEP_SUMMARY}" | |
| awk '/\|Tasks\|Version\|/,/^$/ { if (NF > 0) print }' atom_accuracy_output.txt >> "${GITHUB_STEP_SUMMARY}" || true | |
| - name: Collect ATOM logs | |
| if: always() | |
| run: | | |
| docker cp "${CONTAINER_NAME}:/tmp/atom_server.log" atom_server.log 2>/dev/null || true | |
| docker cp "${CONTAINER_NAME}:/tmp/atom_client.log" atom_client.log 2>/dev/null || true | |
| - name: Upload logs and accuracy results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: flydsl-atom-${{ matrix.model_name }} | |
| path: | | |
| atom_accuracy_output.txt | |
| atom_server.log | |
| atom_client.log | |
| atom-upstream/accuracy_test_results/*.json | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| docker exec "${CONTAINER_NAME}" bash -lc " | |
| set -euo pipefail | |
| cd /workspace/atom-upstream | |
| bash .github/scripts/atom_test.sh stop || true | |
| " || true | |
| docker stop "${CONTAINER_NAME}" || true | |
| docker rm "${CONTAINER_NAME}" || true |