ete_test_gpu-qwen3-5-rl-lmdeploy-grpo-sp4-tp2 #249
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ete_test_gpu | |
| run-name: ete_test_gpu-${{ github.event.inputs.run_case || 'all' }} | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| repo_org: | |
| required: false | |
| description: 'Tested repository organization name. Default is InternLM' | |
| type: string | |
| default: 'InternLM/xtuner' | |
| repo_ref: | |
| required: false | |
| description: 'Set branch or tag or commit id. Default is "main"' | |
| type: string | |
| default: 'main' | |
| run_case: | |
| required: false | |
| description: >- | |
| Case name(s) from autotest/config.yaml. Use "all" for every case, | |
| one name (e.g. "qwen3-rl-lmdeploy"), or comma-separated names | |
| (e.g. "qwen3-5-rl-vl-lmdeploy-grpo,s2-preview-rl-vl-lmdeploy-grpo"). | |
| type: string | |
| default: 'all' | |
| parallel: | |
| required: false | |
| description: >- | |
| pytest-xdist worker count (-n). Cases from config.yaml run in parallel. | |
| Use a number (e.g. 4), or "all" for one worker per selected case. | |
| Resume phases within one case stay sequential. | |
| type: string | |
| default: '4' | |
| CI_E2E_DEBUG: | |
| required: false | |
| description: >- | |
| CI_E2E_DEBUG:if true, start "sleep inf &" after install and before training so the | |
| cluster job stays alive for interactive debug (including after OOM). | |
| Default false. | |
| type: boolean | |
| default: false | |
| env: | |
| CI_GPU_IMAGE_REGISTRY: ${{ secrets.CI_GPU_IMAGE_REGISTRY }} | |
| # ETE PNG reports: llmcitester/xtuner @ branch reports, under ete/{run_id}/ | |
| CI_REPORTS_REPOSITORY: ${{ vars.CI_REPORTS_REPOSITORY || 'llmcitester/xtuner' }} | |
| CI_REPORTS_BRANCH: ${{ vars.CI_REPORTS_BRANCH || 'reports' }} | |
| CI_REPORTS_RAW_URL_BASE: ${{ vars.CI_REPORTS_RAW_URL_BASE || 'https://raw.githubusercontent.com/llmcitester/xtuner/reports/ete' }} | |
| CI_REPORTS_BROWSE_URL_BASE: ${{ vars.CI_REPORTS_BROWSE_URL_BASE || 'https://github.com/llmcitester/xtuner/tree/reports/ete' }} | |
| CI_REPORTS_KEEP_RUNS: ${{ vars.CI_REPORTS_KEEP_RUNS || '30' }} | |
| jobs: | |
| ete_test: | |
| if: ${{!cancelled() }} | |
| runs-on: [h_cluster_ete] | |
| timeout-minutes: 1440 | |
| steps: | |
| - name: Clean workdir | |
| run: sudo git clean -ffdx | |
| - name: Clone repository | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: ${{ github.event.inputs.repo_org || 'InternLM/xtuner' }} | |
| ref: ${{github.event.inputs.repo_ref || 'main'}} | |
| - name: Init job summary | |
| run: | | |
| { | |
| echo "## ETE test report" | |
| echo "" | |
| echo "- Workflow run: [#${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" | |
| echo "- Reports: [ete/${{ github.run_id }}](${{ env.CI_REPORTS_BROWSE_URL_BASE }}/${{ github.run_id }})" | |
| echo "" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: run-test | |
| env: | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| CI_REPORTS_RAW_URL_BASE: ${{ env.CI_REPORTS_RAW_URL_BASE }} | |
| CI_E2E_DEBUG: ${{ github.event.inputs.CI_E2E_DEBUG || 'false' }} | |
| run: | | |
| source /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/miniconda3/bin/activate | |
| conda activate clusterx | |
| conda env list | |
| unset HTTP_PROXY;unset HTTPS_PROXY;unset http_proxy;unset https_proxy; | |
| CASE_INPUT="${{ github.event.inputs.run_case || 'all' }}" | |
| PARALLEL_INPUT="${{ github.event.inputs.parallel || '4' }}" | |
| PYTEST_NODES=() | |
| if [ "$CASE_INPUT" = "all" ]; then | |
| mapfile -t CASE_NAMES < <(python -c 'import yaml; cfg=yaml.safe_load(open("autotest/config.yaml",encoding="utf-8")); print("\n".join(cfg["case"]))') | |
| for case_name in "${CASE_NAMES[@]}"; do | |
| [ -z "$case_name" ] && continue | |
| PYTEST_NODES+=("autotest/test_all.py::test_all[${case_name}]") | |
| done | |
| else | |
| # Support one or more cases: "case_a" or "case_a,case_b, case_c" | |
| IFS=',' read -ra RAW_CASES <<< "$CASE_INPUT" | |
| for raw in "${RAW_CASES[@]}"; do | |
| case_name="$(echo "$raw" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" | |
| [ -z "$case_name" ] && continue | |
| PYTEST_NODES+=("autotest/test_all.py::test_all[${case_name}]") | |
| done | |
| fi | |
| if [ "${#PYTEST_NODES[@]}" -eq 0 ]; then | |
| echo "No valid case names parsed from run_case='${CASE_INPUT}'" | |
| exit 1 | |
| fi | |
| CASE_COUNT="${#PYTEST_NODES[@]}" | |
| if [ "$PARALLEL_INPUT" = "all" ]; then | |
| PARALLEL="$CASE_COUNT" | |
| elif [[ "$PARALLEL_INPUT" =~ ^[1-9][0-9]*$ ]]; then | |
| PARALLEL="$PARALLEL_INPUT" | |
| else | |
| echo "Invalid parallel='${PARALLEL_INPUT}', fallback to 4" | |
| PARALLEL=4 | |
| fi | |
| if [ "$PARALLEL" -gt "$CASE_COUNT" ]; then | |
| PARALLEL="$CASE_COUNT" | |
| fi | |
| echo "Running ${CASE_COUNT} case(s) with pytest-xdist -n ${PARALLEL}" | |
| echo "Cases: ${PYTEST_NODES[*]}" | |
| pytest "${PYTEST_NODES[@]}" -m all -n "${PARALLEL}" -s -vv --run_id ${{ github.run_id }} | |
| - name: Check report files | |
| if: always() | |
| id: reports | |
| run: | | |
| REPORT_DIR="${{ github.workspace }}/${{ github.run_id }}" | |
| if compgen -G "${REPORT_DIR}/*.png" > /dev/null; then | |
| echo "has_reports=true" >> "$GITHUB_OUTPUT" | |
| echo "Found $(ls -1 "${REPORT_DIR}"/*.png | wc -l) comparison plot(s)." | |
| else | |
| echo "has_reports=false" >> "$GITHUB_OUTPUT" | |
| echo "No comparison plots found under ${REPORT_DIR}, skip publish steps." | |
| fi | |
| - name: Upload comparison plots | |
| if: always() && steps.reports.outputs.has_reports == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ${{ github.workspace }}/${{ github.run_id }}/*.png | |
| if-no-files-found: error | |
| retention-days: 7 | |
| name: xtuner-ete-${{ github.run_id }} | |
| - name: Publish and prune ete reports | |
| if: always() && steps.reports.outputs.has_reports == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.CI_REPORTS_DEPLOY_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| REPO="${{ env.CI_REPORTS_REPOSITORY }}" | |
| BRANCH="${{ env.CI_REPORTS_BRANCH }}" | |
| KEEP="${{ env.CI_REPORTS_KEEP_RUNS }}" | |
| SRC="${{ github.workspace }}/${{ github.run_id }}" | |
| DEST="ete/${{ github.run_id }}" | |
| git clone --depth=1 -b "${BRANCH}" "https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" pages | |
| cd pages | |
| rm -rf "${DEST}" | |
| mkdir -p "$(dirname "${DEST}")" | |
| cp -r "${SRC}" "${DEST}" | |
| if [ -d ete ]; then | |
| cd ete | |
| mapfile -t RUN_DIRS < <(find . -maxdepth 1 -type d -regextype posix-egrep -regex './[0-9]+' -printf '%f\n' | sort -rn) | |
| if [ "${#RUN_DIRS[@]}" -gt "${KEEP}" ]; then | |
| for old_run in "${RUN_DIRS[@]:KEEP}"; do | |
| rm -rf "${old_run}" | |
| echo "Removed old report: ${old_run}" | |
| done | |
| fi | |
| cd .. | |
| fi | |
| git add -A | |
| if git diff --staged --quiet; then | |
| echo "No report changes to commit." | |
| else | |
| git -c user.name="github-actions[bot]" -c user.email="github-actions[bot]@users.noreply.github.com" \ | |
| commit -m "chore: publish ete report ${{ github.run_id }}, keep latest ${KEEP} runs" | |
| git push | |
| fi |