eval.yml

name: eval

on:
  pull_request:

permissions:
  contents: read
  copilot-requests: write

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22

      - name: Install pinned CLI dependencies
        run: npm ci

      - name: Lint eval specs
        run: npx vally lint --eval-spec evals

      - name: Run evals
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: npx vally eval --suite pr --output-dir vally-results --junit

      - name: Upload eval results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: vally-results
          path: vally-results