ci.yaml

name: CI

on:
  pull_request:
    types: [opened, synchronize, reopened]

permissions: read-all

jobs:
  validate-branch:
    uses: ./.github/workflows/merge-branch-check.yaml

  jql-codegen:
    name: JQL generated contract
    needs: [validate-branch]
    if: needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped'
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Regenerate from checked-in public contracts
        run: python scripts/generate_jql.py

      - name: Check generated JQL output is current
        run: |
          git diff --exit-code -- \
            src/judgeval/jql/_generated_contract.py \
            src/judgeval/jql/_generated_transport.py

      - name: Build distributions
        run: |
          python scripts/check_distribution_contents.py --plant
          pip install uv
          uv build

      - name: Check distribution contents
        run: python scripts/check_distribution_contents.py

  jql-parity:
    name: JQL parity with judgment-mono
    needs: [validate-branch]
    if: (needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped') && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.actor, '[bot]')
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: judgeval

    steps:
      - name: Checkout SDK
        uses: actions/checkout@v4
        with:
          path: judgeval

      - name: Checkout canonical JQL sources
        uses: actions/checkout@v4
        with:
          repository: JudgmentLabs/judgment-mono
          ref: main
          token: ${{ secrets.RELEASE_BOT_GITHUB_PAT }}
          persist-credentials: false
          path: judgment-mono

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Regenerate from judgment-mono
        run: |
          python scripts/generate_jql.py \
            ../judgment-mono/services/data-access-service/openapi.json \
            ../judgment-mono/services/judgeval-server/openapi.public-jql.json

      - name: Check canonical JQL parity
        run: |
          git diff --exit-code -- \
            src/judgeval/jql/_generated_contract.py \
            src/judgeval/jql/_generated_transport.py

  run-tests:
    needs: [validate-branch]
    if: needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped'
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
        python-version:
          - "3.10"
          - "3.11"
          - "3.12"
          - "3.13"
    name: Unit Tests
    runs-on: ${{ matrix.os }}
    env:
      PYTHONPATH: "."
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
      JUDGMENT_DEV: true

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          pip install uv
          uv sync --dev

      - name: Install Claude Code CLI
        run: |
          npm install -g @anthropic-ai/claude-code

      - name: Run tests
        run: |
          cd src
          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
          uv run pytest tests -n auto --cov=judgeval --cov-report=term-missing

      - name: Report coverage
        if: always()
        working-directory: src
        run: uv run coverage report --format=markdown >> $GITHUB_STEP_SUMMARY

  run-e2e-tests:
    needs: [validate-branch]
    if: "(github.base_ref == 'staging' || github.base_ref == 'main') && !contains(github.actor, '[bot]') && (needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped')"
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13"]
    name: E2E Tests
    runs-on: ubuntu-latest
    env:
      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GEMINI_API_KEY }}
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-1

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set env based on branch
        run: |
          if [ "${{ github.base_ref }}" = "main" ]; then
            echo "TARGET_ENV=main" >> "$GITHUB_ENV"
            echo "BASE_URL=https://api.judgmentlabs.ai" >> "$GITHUB_ENV"
            echo "SECRETS_PATH=prod/api-keys/e2e-tests" >> "$GITHUB_ENV"
            echo "COVERAGE_ARTIFACT=coverage-html-production-${{ matrix.python-version }}" >> "$GITHUB_ENV"
          else
            echo "TARGET_ENV=staging" >> "$GITHUB_ENV"
            echo "BASE_URL=https://staging.api.judgmentlabs.ai" >> "$GITHUB_ENV"
            echo "SECRETS_PATH=stg/api-keys/e2e-tests" >> "$GITHUB_ENV"
            echo "COVERAGE_ARTIFACT=coverage-html-staging-${{ matrix.python-version }}" >> "$GITHUB_ENV"
          fi

      - name: Restore uv cache
        uses: actions/cache/restore@v4
        id: restore-uv-cache
        with:
          path: ~/.cache/uv/
          key: ${{ runner.os }}-uv-judgment-${{ hashFiles('./**/uv.lock') }}
          restore-keys: |
            ${{ runner.os }}-uv-judgment-
            ${{ runner.os }}-uv-

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install judgeval dependencies
        run: |
          pip install uv
          uv sync --dev

      - name: Check if server is running
        run: |
          if ! curl -s "$BASE_URL/health" > /dev/null; then
            echo "Judgment server ($BASE_URL) is not running properly. Check CloudWatch logs."
            exit 1
          else
            echo "Server is running."
          fi
        
      - name: Run E2E tests
        working-directory: src
        run: |
          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id "$SECRETS_PATH" --query SecretString --output text)
          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
          export JUDGMENT_API_URL="$BASE_URL"
          timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest -n auto --dist=loadfile --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests

      - name: Report coverage
        if: always()
        working-directory: src
        run: uv run coverage report --format=markdown >> $GITHUB_STEP_SUMMARY

      - name: Upload coverage HTML report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.COVERAGE_ARTIFACT }}
          path: src/htmlcov

      - name: Save uv cache
        uses: actions/cache/save@v4
        if: always() && steps.restore-uv-cache.outputs.cache-hit != 'true'
        with:
          path: ~/.cache/uv/
          key: ${{ runner.os }}-uv-judgment-${{ hashFiles('./**/uv.lock') }}