Backwards Compatibility Tests #108
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). | |
| # All rights reserved. | |
| # | |
| # SPDX-License-Identifier: BSD-3-Clause | |
| # Nightly canary: runs the IsaacLab test suite against multiple pinned | |
| # IsaacSim versions to catch backwards-compatibility regressions. | |
| # | |
| # Caveats for editors of this file or its action dependencies | |
| # (`.github/actions/docker-build`, `run-tests`, `combine-results`): | |
| # | |
| # * No `pull_request:` trigger — changes are not validated automatically | |
| # at PR time. Manually trigger against the PR branch before merge: | |
| # gh workflow run "Backwards Compatibility Tests" --ref <branch> | |
| # Otherwise breakage surfaces only on the next nightly cron after merge. | |
| # | |
| # * The build steps below pass `cache-from: type=gha` / `cache-to: | |
| # type=gha,mode=max` to docker-build explicitly. Future migration to | |
| # `./.github/actions/ecr-build-push-pull` would share build.yaml's | |
| # cross-runner ECR layer cache, but that's deferred until daily-compat | |
| # is ready to wire ECR auth. | |
| name: Backwards Compatibility Tests | |
| on: | |
| schedule: | |
| # Run daily at 8 PM PST (4 AM UTC) | |
| - cron: '0 4 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| isaacsim_version: | |
| description: 'IsaacSim version image tag to test' | |
| required: true | |
| default: '4.5.0' | |
| type: string | |
| # Concurrency control to prevent parallel runs | |
| concurrency: | |
| group: compatibility-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| jobs: | |
| config: | |
| name: Load Config | |
| runs-on: ubuntu-latest | |
| outputs: | |
| isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} | |
| isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} | |
| isaaclab_image_name: ${{ steps.load.outputs.isaaclab_image_name }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| sparse-checkout: .github/workflows/config.yaml | |
| sparse-checkout-cone-mode: false | |
| - id: load | |
| run: | | |
| set -euo pipefail | |
| f=.github/workflows/config.yaml | |
| echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" | |
| echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" | |
| echo "isaaclab_image_name=$(yq -r .isaaclab_image_name "$f")" >> "$GITHUB_OUTPUT" | |
| setup-versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| versions: ${{ steps.set-versions.outputs.versions }} | |
| steps: | |
| - name: Set Isaac Sim Versions | |
| id: set-versions | |
| run: | | |
| # Define all versions to test in one place | |
| DEFAULT_VERSIONS='["4.5.0", "5.0.0"]' | |
| if [ -n "${{ github.event.inputs.isaacsim_version }}" ]; then | |
| # If a specific version is provided via workflow_dispatch, use only that | |
| echo "versions=[\"${{ github.event.inputs.isaacsim_version }}\"]" >> $GITHUB_OUTPUT | |
| else | |
| # Otherwise, use all default versions | |
| echo "versions=$DEFAULT_VERSIONS" >> $GITHUB_OUTPUT | |
| fi | |
| test-isaaclab-tasks-compat: | |
| needs: [setup-versions, config] | |
| runs-on: [self-hosted, gpu] | |
| timeout-minutes: 180 | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| isaacsim_version: ${{ fromJson(needs.setup-versions.outputs.versions) }} | |
| fail-fast: false | |
| env: | |
| CUDA_VISIBLE_DEVICES: all | |
| NVIDIA_VISIBLE_DEVICES: all | |
| NVIDIA_DRIVER_CAPABILITIES: all | |
| CUDA_HOME: /usr/local/cuda | |
| LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64 | |
| DOCKER_IMAGE_TAG: isaac-lab-compat:${{ github.ref_name }}-${{ github.sha }}-${{ matrix.isaacsim_version }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| lfs: true | |
| # Single source of truth: read the OV runtime pins from [tool.isaaclab.versions]. | |
| - name: Resolve OV runtime pins from pyproject | |
| uses: ./.github/actions/resolve-ov-pins | |
| id: ov_pins | |
| - name: Build Docker Image | |
| uses: ./.github/actions/docker-build | |
| with: | |
| image-tag: ${{ env.DOCKER_IMAGE_TAG }} | |
| isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} | |
| isaacsim-version: ${{ matrix.isaacsim_version }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Run IsaacLab Tasks Tests | |
| uses: ./.github/actions/run-tests | |
| with: | |
| test-path: "tools" | |
| result-file: "isaaclab-tasks-compat-report.xml" | |
| container-name: "isaac-lab-tasks-compat-test-$$" | |
| image-tag: ${{ env.DOCKER_IMAGE_TAG }} | |
| pytest-options: "" | |
| filter-pattern: "isaaclab_tasks" | |
| extra-pip-packages: ${{ format('{0} {1}', steps.ov_pins.outputs.ovphysx, steps.ov_pins.outputs.ovrtx) }} | |
| - name: Copy All Test Results from IsaacLab Tasks Container | |
| run: | | |
| CONTAINER_NAME="isaac-lab-tasks-compat-test-$$" | |
| if docker ps -a | grep -q $CONTAINER_NAME; then | |
| echo "Copying all test results from IsaacLab Tasks container..." | |
| docker cp $CONTAINER_NAME:/workspace/isaaclab/tests/isaaclab-tasks-compat-report.xml reports/ 2>/dev/null || echo "No test results to copy from IsaacLab Tasks container" | |
| fi | |
| - name: Upload IsaacLab Tasks Test Results | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: isaaclab-tasks-compat-results-${{ matrix.isaacsim_version }} | |
| path: reports/isaaclab-tasks-compat-report.xml | |
| retention-days: 7 | |
| compression-level: 9 | |
| test-general-compat: | |
| needs: [setup-versions, config] | |
| runs-on: [self-hosted, gpu] | |
| timeout-minutes: 180 | |
| strategy: | |
| matrix: | |
| isaacsim_version: ${{ fromJson(needs.setup-versions.outputs.versions) }} | |
| fail-fast: false | |
| env: | |
| CUDA_VISIBLE_DEVICES: all | |
| NVIDIA_VISIBLE_DEVICES: all | |
| NVIDIA_DRIVER_CAPABILITIES: all | |
| CUDA_HOME: /usr/local/cuda | |
| LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64 | |
| DOCKER_IMAGE_TAG: isaac-lab-compat:${{ github.ref_name }}-${{ github.sha }}-${{ matrix.isaacsim_version }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| lfs: true | |
| - name: Build Docker Image | |
| uses: ./.github/actions/docker-build | |
| with: | |
| image-tag: ${{ env.DOCKER_IMAGE_TAG }} | |
| isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} | |
| isaacsim-version: ${{ matrix.isaacsim_version }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Run General Tests | |
| uses: ./.github/actions/run-tests | |
| with: | |
| test-path: "tools" | |
| result-file: "general-tests-compat-report.xml" | |
| container-name: "isaac-lab-general-compat-test-$$" | |
| image-tag: ${{ env.DOCKER_IMAGE_TAG }} | |
| pytest-options: "" | |
| filter-pattern: "not isaaclab_tasks" | |
| - name: Copy All Test Results from General Tests Container | |
| run: | | |
| CONTAINER_NAME="isaac-lab-general-compat-test-$$" | |
| if docker ps -a | grep -q $CONTAINER_NAME; then | |
| echo "Copying all test results from General Tests container..." | |
| docker cp $CONTAINER_NAME:/workspace/isaaclab/tests/general-tests-compat-report.xml reports/ 2>/dev/null || echo "No test results to copy from General Tests container" | |
| fi | |
| - name: Upload General Test Results | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: general-tests-compat-results-${{ matrix.isaacsim_version }} | |
| path: reports/general-tests-compat-report.xml | |
| retention-days: 7 | |
| compression-level: 9 | |
| combine-compat-results: | |
| needs: [test-isaaclab-tasks-compat, test-general-compat] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| lfs: false | |
| sparse-checkout: .github/actions | |
| sparse-checkout-cone-mode: true | |
| - name: Create Reports Directory | |
| run: | | |
| mkdir -p reports | |
| - name: Download All Test Results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: '*-compat-results-*' | |
| path: reports/ | |
| merge-multiple: true | |
| continue-on-error: true | |
| - name: Combine All Test Results | |
| uses: ./.github/actions/combine-results | |
| with: | |
| tests-dir: "reports" | |
| output-file: "reports/combined-compat-results.xml" | |
| - name: Upload Combined Test Results | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: daily-compat-${{ github.run_id }}-combined-test-results | |
| path: reports/combined-compat-results.xml | |
| retention-days: 30 | |
| compression-level: 9 | |
| - name: Report Test Results | |
| uses: dorny/test-reporter@v2.6.0 | |
| if: always() | |
| with: | |
| name: IsaacLab Compatibility Test Results (${{ github.event_name }}) | |
| path: reports/combined-compat-results.xml | |
| reporter: java-junit | |
| max-annotations: '50' | |
| report-title: "IsaacLab Compatibility Test Results - ${{ github.event_name }} - ${{ github.ref_name }}" | |
| notify-compatibility-status: | |
| needs: [setup-versions, combine-compat-results] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| lfs: false | |
| sparse-checkout: .github/actions | |
| sparse-checkout-cone-mode: true | |
| - name: Create Compatibility Report | |
| run: | | |
| TRIGGER_INFO="**Trigger:** ${{ github.event_name }}" | |
| ISAACSIM_VERSIONS="${{ join(fromJson(needs.setup-versions.outputs.versions), ', ') }}" | |
| echo "## Daily Backwards Compatibility Test Results" > compatibility-report.md | |
| echo "" >> compatibility-report.md | |
| echo "$TRIGGER_INFO" >> compatibility-report.md | |
| echo "**IsaacSim Versions Tested:** $ISAACSIM_VERSIONS" >> compatibility-report.md | |
| echo "**Branch:** ${{ github.ref_name }}" >> compatibility-report.md | |
| echo "**Commit:** ${{ github.sha }}" >> compatibility-report.md | |
| echo "**Run ID:** ${{ github.run_id }}" >> compatibility-report.md | |
| echo "" >> compatibility-report.md | |
| echo "### Test Status:" >> compatibility-report.md | |
| echo "- Results: ${{ needs.combine-compat-results.result }}" >> compatibility-report.md | |
| echo "" >> compatibility-report.md | |
| echo "### Artifacts:" >> compatibility-report.md | |
| echo "- [Combined Test Results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> compatibility-report.md | |
| echo "" >> compatibility-report.md | |
| echo "---" >> compatibility-report.md | |
| echo "*This report was generated automatically by the daily compatibility workflow.*" >> compatibility-report.md | |
| - name: Upload Compatibility Report | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: compatibility-report-${{ github.run_id }} | |
| path: compatibility-report.md | |
| retention-days: 30 |