LLM Evaluations #47
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LLM Evaluations | |
| on: | |
| pull_request: | |
| paths: | |
| - 'src/core/**' | |
| - 'src/schema/**' | |
| - 'src/binary/**' | |
| - 'src/evals/**' | |
| - 'benchmarks/**' | |
| push: | |
| branches: [main] | |
| schedule: | |
| # Run weekly on Sunday at midnight | |
| - cron: '0 0 * * 0' | |
| workflow_dispatch: | |
| inputs: | |
| eval_type: | |
| description: 'Type of evaluation to run' | |
| required: true | |
| default: 'smoke' | |
| type: choice | |
| options: | |
| - smoke | |
| - full | |
| jobs: | |
| smoke-test: | |
| name: Smoke Test Evaluations | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' || github.event.inputs.eval_type == 'smoke' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build project | |
| run: npm run build | |
| - name: Run smoke evaluations | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: npm run eval:smoke | |
| continue-on-error: true | |
| - name: Check for regressions | |
| id: regression_check | |
| run: npm run eval:check-regressions | |
| continue-on-error: true | |
| - name: Upload eval results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: smoke-eval-results | |
| path: benchmarks/results/ | |
| retention-days: 30 | |
| - name: Comment PR with results | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| try { | |
| const resultPath = path.join(process.cwd(), 'benchmarks/results/latest.json'); | |
| if (!fs.existsSync(resultPath)) { | |
| console.log('No results file found'); | |
| return; | |
| } | |
| const results = JSON.parse(fs.readFileSync(resultPath, 'utf-8')); | |
| const passed = results.passed ? '✅' : '❌'; | |
| const duration = (results.duration / 1000).toFixed(1); | |
| let comment = `## ${passed} LLM Evaluation Results\n\n`; | |
| comment += `**Duration:** ${duration}s\n`; | |
| comment += `**Status:** ${results.passed ? 'Passed' : 'Failed'}\n\n`; | |
| comment += `### Metrics\n\n`; | |
| comment += `| Model | Exact Match | Token Efficiency |\n`; | |
| comment += `|-------|-------------|------------------|\n`; | |
| for (const [model, metrics] of Object.entries(results.results)) { | |
| const exactMatch = ((metrics.exactMatch || 0) * 100).toFixed(1); | |
| const tokenEff = (metrics.tokenEfficiency || 0).toFixed(0); | |
| comment += `| ${model} | ${exactMatch}% | ${tokenEff} |\n`; | |
| } | |
| if (results.regressions && results.regressions.length > 0) { | |
| comment += `\n### ⚠️ Regressions Detected\n\n`; | |
| results.regressions.forEach(r => { | |
| const emoji = r.severity === 'critical' ? '🔴' : r.severity === 'major' ? '🟠' : '🟡'; | |
| comment += `${emoji} **${r.model}/${r.metric}:** ${r.change.toFixed(1)}% change\n`; | |
| }); | |
| } | |
| await github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); | |
| } catch (error) { | |
| console.error('Error posting comment:', error); | |
| } | |
| full-evaluation: | |
| name: Full Evaluation Suite | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' || github.event_name == 'schedule' || github.event.inputs.eval_type == 'full' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build project | |
| run: npm run build | |
| - name: Run full evaluation suite | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: npm run eval:full | |
| timeout-minutes: 30 | |
| - name: Check for regressions | |
| id: regression_check | |
| run: npm run eval:check-regressions -- --type=full | |
| continue-on-error: true | |
| - name: Save as new baseline | |
| if: github.ref == 'refs/heads/main' && steps.regression_check.outcome == 'success' | |
| run: npm run eval:baseline -- --type=full | |
| - name: Upload eval results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: full-eval-results | |
| path: benchmarks/results/ | |
| retention-days: 90 | |
| - name: Fail on critical regressions | |
| if: steps.regression_check.outcome == 'failure' | |
| run: exit 1 |