Golden Eval #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Golden Eval | |
| on: | |
| schedule: | |
| - cron: "0 15 * * 1" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| golden-eval: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10.32.1 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - run: pnpm eval | |
| - name: Open issue on accuracy regression | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| node <<'NODE' | |
| const fs = require('node:fs'); | |
| const path = require('node:path'); | |
| const dir = 'eval-results'; | |
| const latest = fs.readdirSync(dir) | |
| .filter((file) => file.endsWith('.json')) | |
| .sort() | |
| .at(-1); | |
| if (!latest) throw new Error('No eval report generated'); | |
| const report = JSON.parse(fs.readFileSync(path.join(dir, latest), 'utf8')); | |
| const accuracy = report.metrics.exactTierAccuracy; | |
| const threshold = 0.95; | |
| if (accuracy >= threshold) process.exit(0); | |
| fs.writeFileSync( | |
| 'eval-regression.md', | |
| [ | |
| 'Golden-set eval accuracy dropped below the 95% threshold.', | |
| '', | |
| `Report: ${latest}`, | |
| `Accuracy: ${(accuracy * 100).toFixed(1)}%`, | |
| `Candidates: ${report.metrics.candidateCount}`, | |
| `Hallucination rate: ${(report.metrics.hallucinationRate * 100).toFixed(1)}%`, | |
| `Cost: $${report.metrics.totalCostUsd.toFixed(4)}`, | |
| '', | |
| 'Review the generated eval report in the workflow artifacts/logs and compare against the previous weekly run.', | |
| ].join('\n'), | |
| ); | |
| NODE | |
| - name: Create regression issue | |
| if: ${{ hashFiles('eval-regression.md') != '' }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh issue create \ | |
| --title "Golden eval accuracy regression" \ | |
| --body-file eval-regression.md \ | |
| --label "tech-debt" |