daily-compatibility.yml

# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause

# Nightly canary: runs the IsaacLab test suite against multiple pinned
# IsaacSim versions to catch backwards-compatibility regressions.
#
# Caveats for editors of this file or its action dependencies
# (`.github/actions/docker-build`, `run-tests`, `combine-results`):
#
#   * No `pull_request:` trigger — changes are not validated automatically
#     at PR time. Manually trigger against the PR branch before merge:
#         gh workflow run "Backwards Compatibility Tests" --ref <branch>
#     Otherwise breakage surfaces only on the next nightly cron after merge.
#
#   * The build steps below pass `cache-from: type=gha` / `cache-to:
#     type=gha,mode=max` to docker-build explicitly. Future migration to
#     `./.github/actions/ecr-build-push-pull` would share build.yaml's
#     cross-runner ECR layer cache, but that's deferred until daily-compat
#     is ready to wire ECR auth.

name: Backwards Compatibility Tests

on:
  schedule:
    # Run daily at 8 PM PST (4 AM UTC)
    - cron: '0 4 * * *'

  workflow_dispatch:
    inputs:
      isaacsim_version:
        description: 'IsaacSim version image tag to test'
        required: true
        default: '4.5.0'
        type: string

# Concurrency control to prevent parallel runs
concurrency:
  group: compatibility-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read
  pull-requests: write

env:
  NGC_API_KEY: ${{ secrets.NGC_API_KEY }}

jobs:
  config:
    name: Load Config
    runs-on: ubuntu-latest
    outputs:
      isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }}
      isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }}
      isaaclab_image_name: ${{ steps.load.outputs.isaaclab_image_name }}
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        sparse-checkout: .github/workflows/config.yaml
        sparse-checkout-cone-mode: false
    - id: load
      run: |
        set -euo pipefail
        f=.github/workflows/config.yaml
        echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT"
        echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT"
        echo "isaaclab_image_name=$(yq -r .isaaclab_image_name "$f")" >> "$GITHUB_OUTPUT"

  setup-versions:
    runs-on: ubuntu-latest
    outputs:
      versions: ${{ steps.set-versions.outputs.versions }}
    steps:
      - name: Set Isaac Sim Versions
        id: set-versions
        run: |
          # Define all versions to test in one place
          DEFAULT_VERSIONS='["4.5.0", "5.0.0"]'

          if [ -n "${{ github.event.inputs.isaacsim_version }}" ]; then
            # If a specific version is provided via workflow_dispatch, use only that
            echo "versions=[\"${{ github.event.inputs.isaacsim_version }}\"]" >> $GITHUB_OUTPUT
          else
            # Otherwise, use all default versions
            echo "versions=$DEFAULT_VERSIONS" >> $GITHUB_OUTPUT
          fi

  test-isaaclab-tasks-compat:
    needs: [setup-versions, config]
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    continue-on-error: true
    strategy:
      matrix:
        isaacsim_version: ${{ fromJson(needs.setup-versions.outputs.versions) }}
      fail-fast: false
    env:
        CUDA_VISIBLE_DEVICES: all
        NVIDIA_VISIBLE_DEVICES: all
        NVIDIA_DRIVER_CAPABILITIES: all
        CUDA_HOME: /usr/local/cuda
        LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
        DOCKER_IMAGE_TAG: isaac-lab-compat:${{ github.ref_name }}-${{ github.sha }}-${{ matrix.isaacsim_version }}

    steps:
    - name: Checkout Code
      uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true

    # Single source of truth: read the OV runtime pins from [tool.isaaclab.versions].
    - name: Resolve OV runtime pins from pyproject
      uses: ./.github/actions/resolve-ov-pins
      id: ov_pins

    - name: Build Docker Image
      uses: ./.github/actions/docker-build
      with:
        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ matrix.isaacsim_version }}
        cache-from: type=gha
        cache-to: type=gha,mode=max

    - name: Run IsaacLab Tasks Tests
      uses: ./.github/actions/run-tests
      with:
        test-path: "tools"
        result-file: "isaaclab-tasks-compat-report.xml"
        container-name: "isaac-lab-tasks-compat-test-$$"
        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
        pytest-options: ""
        filter-pattern: "isaaclab_tasks"
        extra-pip-packages: ${{ format('{0} {1}', steps.ov_pins.outputs.ovphysx, steps.ov_pins.outputs.ovrtx) }}

    - name: Copy All Test Results from IsaacLab Tasks Container
      run: |
        CONTAINER_NAME="isaac-lab-tasks-compat-test-$$"
        if docker ps -a | grep -q $CONTAINER_NAME; then
          echo "Copying all test results from IsaacLab Tasks container..."
          docker cp $CONTAINER_NAME:/workspace/isaaclab/tests/isaaclab-tasks-compat-report.xml reports/ 2>/dev/null || echo "No test results to copy from IsaacLab Tasks container"
        fi

    - name: Upload IsaacLab Tasks Test Results
      uses: actions/upload-artifact@v7
      if: always()
      with:
        name: isaaclab-tasks-compat-results-${{ matrix.isaacsim_version }}
        path: reports/isaaclab-tasks-compat-report.xml
        retention-days: 7
        compression-level: 9

  test-general-compat:
    needs: [setup-versions, config]
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    strategy:
      matrix:
        isaacsim_version: ${{ fromJson(needs.setup-versions.outputs.versions) }}
      fail-fast: false
    env:
        CUDA_VISIBLE_DEVICES: all
        NVIDIA_VISIBLE_DEVICES: all
        NVIDIA_DRIVER_CAPABILITIES: all
        CUDA_HOME: /usr/local/cuda
        LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
        DOCKER_IMAGE_TAG: isaac-lab-compat:${{ github.ref_name }}-${{ github.sha }}-${{ matrix.isaacsim_version }}

    steps:
    - name: Checkout Code
      uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true

    - name: Build Docker Image
      uses: ./.github/actions/docker-build
      with:
        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ matrix.isaacsim_version }}
        cache-from: type=gha
        cache-to: type=gha,mode=max

    - name: Run General Tests
      uses: ./.github/actions/run-tests
      with:
        test-path: "tools"
        result-file: "general-tests-compat-report.xml"
        container-name: "isaac-lab-general-compat-test-$$"
        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
        pytest-options: ""
        filter-pattern: "not isaaclab_tasks"

    - name: Copy All Test Results from General Tests Container
      run: |
        CONTAINER_NAME="isaac-lab-general-compat-test-$$"
        if docker ps -a | grep -q $CONTAINER_NAME; then
          echo "Copying all test results from General Tests container..."
          docker cp $CONTAINER_NAME:/workspace/isaaclab/tests/general-tests-compat-report.xml reports/ 2>/dev/null || echo "No test results to copy from General Tests container"
        fi

    - name: Upload General Test Results
      uses: actions/upload-artifact@v7
      if: always()
      with:
        name: general-tests-compat-results-${{ matrix.isaacsim_version }}
        path: reports/general-tests-compat-report.xml
        retention-days: 7
        compression-level: 9

  combine-compat-results:
    needs: [test-isaaclab-tasks-compat, test-general-compat]
    runs-on: ubuntu-latest
    if: always()

    steps:
    - name: Checkout Code
      uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: false
        sparse-checkout: .github/actions
        sparse-checkout-cone-mode: true

    - name: Create Reports Directory
      run: |
        mkdir -p reports

    - name: Download All Test Results
      uses: actions/download-artifact@v8
      with:
        pattern: '*-compat-results-*'
        path: reports/
        merge-multiple: true
      continue-on-error: true

    - name: Combine All Test Results
      uses: ./.github/actions/combine-results
      with:
        tests-dir: "reports"
        output-file: "reports/combined-compat-results.xml"

    - name: Upload Combined Test Results
      uses: actions/upload-artifact@v7
      if: always()
      with:
        name: daily-compat-${{ github.run_id }}-combined-test-results
        path: reports/combined-compat-results.xml
        retention-days: 30
        compression-level: 9

    - name: Report Test Results
      uses: dorny/test-reporter@v2.6.0
      if: always()
      with:
        name: IsaacLab Compatibility Test Results (${{ github.event_name }})
        path: reports/combined-compat-results.xml
        reporter: java-junit
        max-annotations: '50'
        report-title: "IsaacLab Compatibility Test Results - ${{ github.event_name }} - ${{ github.ref_name }}"

  notify-compatibility-status:
    needs: [setup-versions, combine-compat-results]
    runs-on: ubuntu-latest
    if: always()

    steps:
    - name: Checkout Code
      uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: false
        sparse-checkout: .github/actions
        sparse-checkout-cone-mode: true

    - name: Create Compatibility Report
      run: |
        TRIGGER_INFO="**Trigger:** ${{ github.event_name }}"
        ISAACSIM_VERSIONS="${{ join(fromJson(needs.setup-versions.outputs.versions), ', ') }}"
        echo "## Daily Backwards Compatibility Test Results" > compatibility-report.md
        echo "" >> compatibility-report.md
        echo "$TRIGGER_INFO" >> compatibility-report.md
        echo "**IsaacSim Versions Tested:** $ISAACSIM_VERSIONS" >> compatibility-report.md
        echo "**Branch:** ${{ github.ref_name }}" >> compatibility-report.md
        echo "**Commit:** ${{ github.sha }}" >> compatibility-report.md
        echo "**Run ID:** ${{ github.run_id }}" >> compatibility-report.md
        echo "" >> compatibility-report.md
        echo "### Test Status:" >> compatibility-report.md
        echo "- Results: ${{ needs.combine-compat-results.result }}" >> compatibility-report.md
        echo "" >> compatibility-report.md
        echo "### Artifacts:" >> compatibility-report.md
        echo "- [Combined Test Results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> compatibility-report.md
        echo "" >> compatibility-report.md
        echo "---" >> compatibility-report.md
        echo "*This report was generated automatically by the daily compatibility workflow.*" >> compatibility-report.md

    - name: Upload Compatibility Report
      uses: actions/upload-artifact@v7
      if: always()
      with:
        name: compatibility-report-${{ github.run_id }}
        path: compatibility-report.md
        retention-days: 30