run_tests.yml

name: Lint, Test, and Build

on:
  push:
    branches:
      - main
  pull_request:
    types: [opened, synchronize, reopened]

permissions: {}

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  workflow-lint:
    name: Lint GitHub Actions Workflows
    runs-on: ubuntu-latest
    permissions:
      contents: read
      security-events: write
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - name: Run zizmor
        uses: zizmorcore/zizmor-action@70fb788f84895a7701f5643d103d587e460b5c99 # v0.6.3
      - name: Run actionlint
        uses: raven-actions/actionlint@3d39aea434753780c3b3d4a1a31c854b4dbf49d7 # v2.2.0

  fix:
    name: Check Ruff Fix
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.11"
      - name: Install uv with caching
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock
      - name: Create and activate virtual environment
        run: |
          uv venv .venv
          echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
      - name: Install dependencies
        run: uv sync --dev -p .venv --extra dev
      - name: Ruff Check
        run: |
          ruff check --fix-only --diff --exit-non-zero-on-fix || (
            echo ""
            echo "❌ Ruff found issues that can be fixed automatically."
            echo "💡 To fix them locally, run:"
            echo ""
            echo "    pre-commit run --all-files"
            echo ""
            echo "Then commit and push the changes."
            exit 1
          )

  test:
    name: Run Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv with caching
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock
      - name: Create and activate virtual environment
        run: |
          uv venv .venv
          echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
      - name: Install dependencies
        run: |
          uv sync --dev -p .venv --extra dev
          uv pip list
      - name: Run tests with pytest
        run: uv run -p .venv pytest -vv -m 'not extra and not deno' -n auto --dist worksteal tests/

  extra_test:
    name: Run Extra and Deno Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv with caching
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock
      - name: Create and activate virtual environment
        run: |
          uv venv .venv
          echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
      - name: Install dependencies
        run: |
          uv sync -p .venv --extra deno --extra dev --extra test_extras
          uv run -p .venv python -c 'import deno, subprocess; subprocess.run([deno.find_deno_bin(), "--version"], check=True)'
      - name: Run extra and deno tests
        run: uv run -p .venv pytest tests/ -m 'extra or deno' --extra --deno -n auto --dist worksteal

  mcp_v2_test:
    name: Run MCP v2 Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.11"
      - name: Install uv with caching
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock
      - name: Create environment
        run: uv venv .venv
      - name: Install dependencies with MCP v2
        run: |
          uv sync -p .venv --extra dev
          uv pip install -p .venv "mcp==2.0.0"
      - name: Run MCP tests
        run: uv run --frozen --no-sync -p .venv pytest tests/utils/test_mcp.py -m extra --extra

  llm_call_test:
    name: Run Tests with Real LM
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: 3.11
      - name: Prefetch Ollama image
        run: |
          (
            if docker pull ollama/ollama:0.31.2 > "$RUNNER_TEMP/ollama-pull.log" 2>&1; then
              echo 0 > "$RUNNER_TEMP/ollama-pull.status"
            else
              echo "$?" > "$RUNNER_TEMP/ollama-pull.status"
            fi
          ) &
      - name: Install uv with caching
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock
      - name: Create and activate virtual environment
        run: |
          uv venv .venv
          echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
      - name: Install dependencies
        run: |
          uv sync --dev -p .venv --extra dev
          uv pip list
      - name: Cache Ollama models
        id: cache-ollama
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ollama-data
          key: ollama-llama3.2-3b-${{ runner.os }}-ollama0.31.2-v2
      - name: Start Ollama service
        run: |
          for _ in {1..120}; do
            [[ -f "$RUNNER_TEMP/ollama-pull.status" ]] && break
            sleep 1
          done
          if [[ ! -f "$RUNNER_TEMP/ollama-pull.status" ]] || [[ "$(cat "$RUNNER_TEMP/ollama-pull.status")" != 0 ]]; then
            cat "$RUNNER_TEMP/ollama-pull.log"
            exit 1
          fi
          mkdir -p ollama-data
          docker run -d --name ollama \
            -p 11434:11434 \
            -v "$GITHUB_WORKSPACE/ollama-data:/root/.ollama" \
            ollama/ollama:0.31.2
          # Some GitHub runners are Sapphire-Rapids-class machines with AMX. llama.cpp's
          # AMX-repacked weight path segfaults there during model load (llama-server dies
          # with SIGSEGV in warm-up), while all non-AMX runners pass. Removing the AMX
          # backend variant makes ggml's autodetection fall back to the icelake kernels
          # that every other runner already uses.
          docker exec ollama rm /usr/lib/ollama/libggml-cpu-sapphirerapids.so
          timeout 60 bash -c 'until curl -f http://localhost:11434/api/version; do sleep 2; done'
      - name: Pull LLM
        if: steps.cache-ollama.outputs.cache-hit != 'true'
        run: docker exec ollama ollama pull llama3.2:3b
      - name: Warm up LLM
        run: |
          for i in 1 2 3; do
            if curl -fsS http://localhost:11434/api/generate \
              -d '{"model":"llama3.2:3b","prompt":"ping","stream":false}'; then
              echo "Ollama warm-up succeeded"
              exit 0
            fi
            echo "Warm-up attempt $i failed; retrying..."
            sleep 5
          done
          echo "Ollama failed to serve llama3.2:3b after 3 attempts"
          docker logs ollama || true
          exit 1
      - name: Set LM environment variable
        run: echo "LM_FOR_TEST=ollama/llama3.2:3b" >> "$GITHUB_ENV"
      - name: Run tests
        run: uv run -p .venv pytest -m llm_call --llm_call -vv --durations=5 tests/
      - name: Fix permissions for cache
        if: always()
        run: sudo chown -R "$USER":"$USER" ollama-data || true
      - name: Stop Ollama service
        if: always()
        run: docker stop ollama && docker rm ollama

  build_package:
    name: Build Package
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv with caching
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock
      - name: Build
        run: uv build
      - name: Install built package
        run: |
          uv venv .venv
          uv pip install dist/*.whl -p .venv
      - name: Test import dspy
        run: .venv/bin/python -c "import dspy"