eval-regression.yml

# Comet skill eval regression gate.
#
# Pull requests only smoke-check the regression gate wiring. Real eval runs can
# call external models and are restricted to manual workflow_dispatch runs.
#
# NOTE: requires Docker (for the subject agent) and an Anthropic-compatible
# API key (ANTHROPIC_AUTH_TOKEN or ANTHROPIC_API_KEY) as a repository secret
# when manually dispatched.

name: eval-regression

on:
  pull_request:
    paths:
      - "assets/skills/**"
      - "assets/skills-zh/**"
      - "eval/**"
  workflow_dispatch:

jobs:
  gate:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install uv
        run: pip install uv

      - name: Install eval deps
        working-directory: eval
        run: uv sync

      - name: Smoke check regression gate wiring
        if: github.event_name == 'pull_request'
        working-directory: eval
        run: |
          uv run python local/scripts/regression_check.py --help

      - name: Set up Docker
        if: github.event_name == 'workflow_dispatch'
        uses: docker/setup-buildx-action@v3

      - name: Run regression gate
        if: github.event_name == 'workflow_dispatch'
        working-directory: eval
        env:
          ANTHROPIC_AUTH_TOKEN: ${{ secrets.ANTHROPIC_AUTH_TOKEN }}
          ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }}
          ANTHROPIC_MODEL: ${{ secrets.ANTHROPIC_MODEL }}
          BENCH_LOOP_MAX_TURNS: "8"
        run: |
          uv run python local/scripts/regression_check.py --count 1 --tolerance 0.10

      - name: Upload experiment logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-logs
          path: eval/local/logs/experiments/