ci.yml

# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

name: IsaacLab Arena CI

on:
  pull_request:
  push:
    branches: [ "main" ]
  workflow_dispatch:

# Concurrency control to prevent parallel runs on the same PR
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  pull-requests: write
  checks: write
  issues: read

env:
  NGC_API_KEY: ${{ secrets.ARENA_NGC_API_KEY }}
  OMNI_PASS: ${{ secrets.OMNI_PASS }}
  OMNI_USER: ${{ secrets.OMNI_USER }}


jobs:

  pre_commit:
    name: Pre-commit
    runs-on: [self-hosted, gpu-arena]
    timeout-minutes: 30

    container:
      image: python:3.11-slim

    env:
      # We're getting issues with the markers on the checked out files.
      SKIP: check-executables-have-shebangs

    steps:
      - &install_git_step
        name: Install git (and tools needed by hooks)
        run: |
          apt-get update
          apt-get install -y --no-install-recommends git git-lfs clang-format ca-certificates make
          git --version
          git lfs version

      # Clean up residual stuff due to symlinks in the submodules directory
      # We do this prior to checkout because the symlink causes issues if it's there from a previous run.
      - &cleanup_step
        name: Clean up symlinks in submodules directory
        run: |
          rm -f .git/modules/submodules/IsaacLab/index.lock || true
          rm -rf submodules/* || true

      # Fix "detected dubious ownership in repository" inside containers
      - &mark_repo_safe_step
        name: Mark repo as safe for git
        run: git config --global --add safe.directory "$PWD"

      # Checkout Code
      - &checkout_step
        name: Checkout Code
        uses: actions/checkout@v4
        with:
          submodules: true
          # LFS checkout here somehow causes issues when LFS stuff changes over time.
          # So I do LFS manually in a step after.
          # lfs: true

      # Pull LFS files explicitly (in case checkout didn't get them all)
      - &git_lfs_step
        name: Git LFS
        run: |
          git lfs update --force
          git lfs install --local
          git lfs pull

      - name: git status
        run: |
          git status

      - name: Run pre-commit
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: --all-files --verbose
        env:
          # Avoid "dubious ownership" errors on restored cache dirs — use a
          # fresh directory owned by the current process instead.
          PRE_COMMIT_HOME: /tmp/pre-commit-cache

  test:
    name: Run tests
    runs-on: [self-hosted, gpu-arena]
    timeout-minutes: 80
    needs: [pre_commit]
    env:
      # NV_API_KEY is consumed by the live-endpoint tests for agentic env gen.
      NV_API_KEY: ${{ secrets.ARENA_NV_API_KEY }}

    container:
      image: nvcr.io/nvstaging/isaac-amr/isaaclab_arena:latest
      credentials:
        username: $oauthtoken
        password: ${{ env.NGC_API_KEY }}

    steps:
      # nvidia-smi
      - &nvidia_smi
        name: nvidia-smi
        run: nvidia-smi

      - &lscpu
        name: lscpu
        run: lscpu

      # Setup
      - *install_git_step
      - *cleanup_step
      - *mark_repo_safe_step
      - *checkout_step
      - *git_lfs_step

      - &install_project_step
        name: Install project and link isaac-sim
        run: |
          /isaac-sim/python.sh -m pip install --no-cache-dir -e .
          [ -e ./submodules/IsaacLab/_isaac_sim ] || ln -s /isaac-sim ./submodules/IsaacLab/_isaac_sim

      - &setup_isaac_sim_kit_cache_step
        name: Setup Isaac Sim kit cache (shader warmup on miss)
        uses: ./.github/actions/setup-isaac-sim-kit-cache

      # Run the tests.
      #
      # Note(alexmillane): We have to run the tests in a specific way. To achieve a reasonable test speed,
      # we don't close the SimulationApp between tests, where this is possible. We need, however, to restart the
      # SimulationApp between tests that require cameras/rendering and those that don't. To make
      # matters more complicated, since Isaac Sim 6.0, this persistent SimulationApp scheme is also not
      # mixing well with subprocess-spawning tests. We get intermittent hangs if a subprocess-spawning
      # test is run while a persistent SimulationApp is still open. For that reason we also need to
      # separate those test that spawn subprocess. So in the end, we run
      #
      # - All tests not requiring cameras in a shared SimulationApp.
      # - All tests requiring cameras in a shared SimulationApp.
      # - All subprocess-spawning tests (each spawns their own SimulationApp)
      #
      # To restore sanity here is a goal for Isaac Lab Arena v0.3.
      - name: Run in-process Newton tests
        run: |
          /isaac-sim/python.sh -m pytest -sv --durations=0 -m with_newton \
            isaaclab_arena/tests/

      - name: Run in-process PhysX tests without cameras
        run: |
          /isaac-sim/python.sh -m pytest -sv --durations=0 -m "not with_cameras and not with_subprocess and not with_newton" \
            isaaclab_arena/tests/

      - name: Run GR00T policy/data tests (lightweight gr00t deps only)
        run: |
          /isaac-sim/python.sh -m pytest -sv --durations=0 -m gr00t_policy \
            isaaclab_arena_gr00t/tests/


  tests_with_subprocess:
    name: Run tests with subprocess
    runs-on: [self-hosted, gpu-arena]
    timeout-minutes: 60
    needs: [pre_commit]

    container:
      image: nvcr.io/nvstaging/isaac-amr/isaaclab_arena:latest
      credentials:
        username: $oauthtoken
        password: ${{ env.NGC_API_KEY }}
    steps:
      # nvidia-smi
      - *nvidia_smi
      - *lscpu

      # Setup.
      - *install_git_step
      - *cleanup_step
      - *mark_repo_safe_step
      - *checkout_step
      - *git_lfs_step
      - *install_project_step
      - *setup_isaac_sim_kit_cache_step

      # NOTE(alexmillane): These tests are unreliable at the moment, and often stall,
      # causing the whole CI run to fail. I am moving these to a parallel job, and then
      # will make the job containing them optional in terms of passing.
      - name: Run subprocess-spawning PhysX tests
        run: |
          ISAACLAB_ARENA_SUBPROCESS_TIMEOUT=900 \
          /isaac-sim/python.sh -m pytest -sv --durations=0 -m with_subprocess \
            isaaclab_arena/tests/


  test_gr00t_closedloop_e2e:
    name: GR00T closed-loop E2E
    runs-on: [self-hosted, gpu-arena]
    timeout-minutes: 60
    needs: [pre_commit]
    env:
      GR00T_REMOTE_HOST: gr00t
      GR00T_REMOTE_PORT: "5555"
      GR00T_REMOTE_E2E_TIMEOUT: "900"

    container:
      image: nvcr.io/nvstaging/isaac-amr/isaaclab_arena:latest
      credentials:
        username: $oauthtoken
        password: ${{ env.NGC_API_KEY }}

    services:
      gr00t:
        image: nvcr.io/nvidian/gr00t1_6_arena_ci:latest
        credentials:
          username: $oauthtoken
          password: ${{ env.NGC_API_KEY }}
        ports:
          - 5555:5555
        # Mount the runner's workspace into the GR00T container. The baked
        # entrypoint polls for isaaclab_arena_gr00t/docker/ci_gr00t_train_and_serve.sh.
        volumes:
          - ${{ github.workspace }}:/arena_workspace
        options: >-
          --gpus all
          --entrypoint /workspace/ci_gr00t_train_and_serve.sh

    steps:
      - *nvidia_smi
      - *lscpu
      - *install_git_step
      - *cleanup_step
      - *mark_repo_safe_step
      - *checkout_step
      - *git_lfs_step
      - *install_project_step
      - *setup_isaac_sim_kit_cache_step

      # The GR00T sidecar post-trains a tiny checkpoint before loading it. Its
      # entrypoint waits for the LFS-backed dataset media fetched above before
      # starting training.
      # Poll the ZMQ endpoint because the socket can bind before the model is ready.
      - name: Wait for GR00T server
        timeout-minutes: 20
        env:
          PYTHONUNBUFFERED: "1"
        run: |
          /isaac-sim/python.sh -u -m isaaclab_arena_gr00t.utils.wait_for_gr00t_server \
            --host "${GR00T_REMOTE_HOST}" \
            --port "${GR00T_REMOTE_PORT}" \
            --timeout-sec 1200 \
            --poll-interval-sec 15 \
            --request-timeout-ms 5000

      - name: Run GR00T remote closed-loop policy runner test
        run: |
          /isaac-sim/python.sh -m pytest -sv --durations=0 -m gr00t_remote_e2e \
            isaaclab_arena_gr00t/tests/test_gr00t_remote_closedloop_policy_runner.py


  build_docs_pre_merge:
    name: Build the docs (pre-merge)
    runs-on: [self-hosted, gpu-arena]
    timeout-minutes: 30
    needs: [pre_commit]

    container:
      image: python:3.11-slim

    steps:
      # Setup.
      - *install_git_step
      - *mark_repo_safe_step
      - *cleanup_step

      - name: Checkout Code
        uses: actions/checkout@v4

      # Build docs for the current branch (validates PR changes)
      - name: Build docs
        working-directory: ./docs
        run: |
          pip3 install -r requirements.txt
          make SPHINXOPTS=-W html


  build_and_push_image_post_merge:
    name: Build & push NGC image (post-merge)
    runs-on: [self-hosted, gpu-arena]
    timeout-minutes: 60
    needs: [test]       # only push if tests passed
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'

    container:
      image: ubuntu:latest

    env:
      # already defined at top, but keep here for clarity/local override if needed
      NGC_API_KEY: ${{ secrets.ARENA_NGC_API_KEY }}

    steps:
      # Setup.
      - *install_git_step
      - *mark_repo_safe_step
      - *cleanup_step
      - *checkout_step

      - name: Install docker
        run: |
          apt-get update
          apt-get install -y ca-certificates curl gnupg && \
          install -m 0755 -d /etc/apt/keyrings && \
          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
          chmod a+r /etc/apt/keyrings/docker.gpg && \
          echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
          "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null &&
          apt-get update && apt-get install -y docker.io docker-buildx-plugin
          docker --version

      - name: Docker login to NGC
        # $oauthtoken is the literal NGC username; pipe the NGC API key via stdin
        run: echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin

      - name: Build & push image to NGC
        run: |
          chmod +x ./docker/push_to_ngc* || true
          ./docker/push_to_ngc.sh -p