llm-evals.yml

name: LLM Evaluations

on:
  pull_request:
    paths:
      - 'src/core/**'
      - 'src/schema/**'
      - 'src/binary/**'
      - 'src/evals/**'
      - 'benchmarks/**'
  push:
    branches: [main]
  schedule:
    # Run weekly on Sunday at midnight
    - cron: '0 0 * * 0'
  workflow_dispatch:
    inputs:
      eval_type:
        description: 'Type of evaluation to run'
        required: true
        default: 'smoke'
        type: choice
        options:
          - smoke
          - full

jobs:
  smoke-test:
    name: Smoke Test Evaluations
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request' || github.event.inputs.eval_type == 'smoke'
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
      
      - name: Install dependencies
        run: npm ci
      
      - name: Build project
        run: npm run build
      
      - name: Run smoke evaluations
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: npm run eval:smoke
        continue-on-error: true
      
      - name: Check for regressions
        id: regression_check
        run: npm run eval:check-regressions
        continue-on-error: true
      
      - name: Upload eval results
        uses: actions/upload-artifact@v4
        with:
          name: smoke-eval-results
          path: benchmarks/results/
          retention-days: 30
      
      - name: Comment PR with results
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const path = require('path');
            
            try {
              const resultPath = path.join(process.cwd(), 'benchmarks/results/latest.json');
              if (!fs.existsSync(resultPath)) {
                console.log('No results file found');
                return;
              }
              
              const results = JSON.parse(fs.readFileSync(resultPath, 'utf-8'));
              
              const passed = results.passed ? '✅' : '❌';
              const duration = (results.duration / 1000).toFixed(1);
              
              let comment = `## ${passed} LLM Evaluation Results\n\n`;
              comment += `**Duration:** ${duration}s\n`;
              comment += `**Status:** ${results.passed ? 'Passed' : 'Failed'}\n\n`;
              
              comment += `### Metrics\n\n`;
              comment += `| Model | Exact Match | Token Efficiency |\n`;
              comment += `|-------|-------------|------------------|\n`;
              
              for (const [model, metrics] of Object.entries(results.results)) {
                const exactMatch = ((metrics.exactMatch || 0) * 100).toFixed(1);
                const tokenEff = (metrics.tokenEfficiency || 0).toFixed(0);
                comment += `| ${model} | ${exactMatch}% | ${tokenEff} |\n`;
              }
              
              if (results.regressions && results.regressions.length > 0) {
                comment += `\n### ⚠️ Regressions Detected\n\n`;
                results.regressions.forEach(r => {
                  const emoji = r.severity === 'critical' ? '🔴' : r.severity === 'major' ? '🟠' : '🟡';
                  comment += `${emoji} **${r.model}/${r.metric}:** ${r.change.toFixed(1)}% change\n`;
                });
              }
              
              await github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: comment
              });
            } catch (error) {
              console.error('Error posting comment:', error);
            }

  full-evaluation:
    name: Full Evaluation Suite
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || github.event_name == 'schedule' || github.event.inputs.eval_type == 'full'
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
      
      - name: Install dependencies
        run: npm ci
      
      - name: Build project
        run: npm run build
      
      - name: Run full evaluation suite
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: npm run eval:full
        timeout-minutes: 30
      
      - name: Check for regressions
        id: regression_check
        run: npm run eval:check-regressions -- --type=full
        continue-on-error: true
      
      - name: Save as new baseline
        if: github.ref == 'refs/heads/main' && steps.regression_check.outcome == 'success'
        run: npm run eval:baseline -- --type=full
      
      - name: Upload eval results
        uses: actions/upload-artifact@v4
        with:
          name: full-eval-results
          path: benchmarks/results/
          retention-days: 90
      
      - name: Fail on critical regressions
        if: steps.regression_check.outcome == 'failure'
        run: exit 1