evals-smoke.yml

name: Evals Smoke

on:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: evals-smoke-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  plan:
    name: Plan changed skills
    runs-on: ubuntu-latest

    outputs:
      run_count: ${{ steps.plan.outputs.run_count }}
      skill_count: ${{ steps.plan.outputs.skill_count }}
      base_ref: ${{ steps.plan.outputs.base_ref }}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install planning dependency
        run: python -m pip install --upgrade pip pyyaml

      - name: Fetch base branch
        if: github.event_name == 'pull_request'
        run: git fetch origin "${{ github.base_ref }}" --depth=1

      - name: Resolve eval plan
        id: plan
        run: |
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            BASE_REF="origin/${{ github.base_ref }}"
          else
            BASE_REF="origin/master"
          fi

          python evals/run_evals.py \
            --changed-only \
            --base-ref "$BASE_REF" \
            --plan-only \
            --selection-json eval-plan.json

          python - <<'PY'
          import json
          import os
          from pathlib import Path

          plan = json.loads(Path("eval-plan.json").read_text())
          with Path(os.environ["GITHUB_OUTPUT"]).open("a") as fh:
              fh.write(f"run_count={plan['run_count']}\n")
              fh.write(f"skill_count={plan['skill_count']}\n")
              fh.write(f"base_ref={plan.get('base_ref') or ''}\n")
          with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as fh:
              fh.write("## Eval plan\n\n")
              fh.write(f"- Base ref: `{plan.get('base_ref') or 'n/a'}`\n")
              fh.write(f"- Skills: {plan['skill_count']}\n")
              fh.write(f"- Scenarios: {plan['scenario_count']}\n")
              fh.write(f"- Runs: {plan['run_count']}\n")
              if plan["skills"]:
                  fh.write(f"- Selected: `{', '.join(plan['skills'])}`\n")
              else:
                  fh.write("- Selected: none\n")
          PY

      - name: Note fork PR limitation
        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
        run: |
          {
            echo
            echo "Model-backed smoke evals are skipped for fork PRs because repository secrets are not exposed to untrusted pull_request runs."
            echo "Maintainers can run the workflow manually with workflow_dispatch after reviewing the branch."
          } >> "$GITHUB_STEP_SUMMARY"

      - uses: actions/upload-artifact@v4
        with:
          name: eval-plan
          path: eval-plan.json

  smoke:
    name: Run smoke evals
    runs-on: ubuntu-latest
    needs: plan
    if: needs.plan.outputs.run_count != '0' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository))
    timeout-minutes: 45

    env:
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

    steps:
      - name: Check ANTHROPIC_API_KEY availability
        id: key
        run: |
          if [ -z "$ANTHROPIC_API_KEY" ]; then
            echo "available=false" >> "$GITHUB_OUTPUT"
            {
              echo "Model-backed smoke evals skipped: the ANTHROPIC_API_KEY repository secret is not configured."
              echo "Configure it to enable smoke evals on internal PRs, or run evals locally (evals/run_evals.py)."
            } >> "$GITHUB_STEP_SUMMARY"
          else
            echo "available=true" >> "$GITHUB_OUTPUT"
          fi

      - uses: actions/checkout@v4
        if: steps.key.outputs.available == 'true'
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        if: steps.key.outputs.available == 'true'
        with:
          python-version: "3.12"

      - name: Install Python dependency
        if: steps.key.outputs.available == 'true'
        run: python -m pip install --upgrade pip pyyaml

      - name: Fetch base branch
        if: github.event_name == 'pull_request' && steps.key.outputs.available == 'true'
        run: git fetch origin "${{ github.base_ref }}" --depth=1

      - name: Restore eval cache
        if: steps.key.outputs.available == 'true'
        id: cache-restore
        uses: actions/cache/restore@v4
        with:
          path: evals-workspace/cache
          key: evals-cache-${{ runner.os }}-${{ github.repository }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
          restore-keys: |
            evals-cache-${{ runner.os }}-${{ github.repository }}-${{ github.base_ref }}-
            evals-cache-${{ runner.os }}-${{ github.repository }}-${{ github.ref_name }}-
            evals-cache-${{ runner.os }}-${{ github.repository }}-

      - name: Install claude CLI
        if: steps.key.outputs.available == 'true'
        run: |
          curl -fsSL https://claude.ai/install.sh | bash
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Verify claude CLI
        if: steps.key.outputs.available == 'true'
        run: |
          export PATH="$HOME/.local/bin:$PATH"
          claude --version

      - name: Run smoke evals
        if: steps.key.outputs.available == 'true'
        run: |
          export PATH="$HOME/.local/bin:$PATH"
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            BASE_REF="origin/${{ github.base_ref }}"
          else
            BASE_REF="origin/master"
          fi

          python evals/run_evals.py \
            --changed-only \
            --base-ref "$BASE_REF" \
            --reuse-cache \
            --selection-json eval-plan.json \
            --iteration "ci-${{ github.run_id }}-${{ github.run_attempt }}" \
            --workers 6

      - name: Write benchmark summary
        if: always() && steps.key.outputs.available == 'true'
        run: |
          python - <<'PY'
          import json
          import os
          from pathlib import Path

          summary = Path(os.environ["GITHUB_STEP_SUMMARY"])
          benchmark_files = sorted(Path("evals-workspace").glob("iteration-ci-*/benchmark.json"))
          if not benchmark_files:
              with summary.open("a") as fh:
                  fh.write("\nNo benchmark artifact was produced.\n")
              raise SystemExit(0)

          benchmark = json.loads(benchmark_files[-1].read_text())
          agg = benchmark["aggregate"]
          with summary.open("a") as fh:
              fh.write("\n## Smoke benchmark\n\n")
              fh.write(f"- With skill: {agg['with_skill']['mean_pass_rate']:.0%}\n")
              fh.write(f"- Without skill: {agg['without_skill']['mean_pass_rate']:.0%}\n")
              fh.write(f"- Delta: {agg['delta']:+.0%}\n")
              fh.write(f"- Cost (with skill): ${agg['with_skill']['total_cost_usd']:.2f}\n")
              fh.write(f"- Cost (without skill): ${agg['without_skill']['total_cost_usd']:.2f}\n")
          PY

      - name: Save eval cache
        if: always() && steps.key.outputs.available == 'true'
        uses: actions/cache/save@v4
        with:
          path: evals-workspace/cache
          key: ${{ steps.cache-restore.outputs.cache-primary-key }}

      - uses: actions/upload-artifact@v4
        if: always() && steps.key.outputs.available == 'true'
        with:
          name: eval-benchmark
          path: |
            eval-plan.json
            evals-workspace/iteration-ci-*/benchmark.json