feat(integrations): add Pennylane transaction fetcher #80
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Evals Smoke | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: evals-smoke-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| plan: | |
| name: Plan changed skills | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run_count: ${{ steps.plan.outputs.run_count }} | |
| skill_count: ${{ steps.plan.outputs.skill_count }} | |
| base_ref: ${{ steps.plan.outputs.base_ref }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install planning dependency | |
| run: python -m pip install --upgrade pip pyyaml | |
| - name: Fetch base branch | |
| if: github.event_name == 'pull_request' | |
| run: git fetch origin "${{ github.base_ref }}" --depth=1 | |
| - name: Resolve eval plan | |
| id: plan | |
| run: | | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| BASE_REF="origin/${{ github.base_ref }}" | |
| else | |
| BASE_REF="origin/master" | |
| fi | |
| python evals/run_evals.py \ | |
| --changed-only \ | |
| --base-ref "$BASE_REF" \ | |
| --plan-only \ | |
| --selection-json eval-plan.json | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| plan = json.loads(Path("eval-plan.json").read_text()) | |
| with Path(os.environ["GITHUB_OUTPUT"]).open("a") as fh: | |
| fh.write(f"run_count={plan['run_count']}\n") | |
| fh.write(f"skill_count={plan['skill_count']}\n") | |
| fh.write(f"base_ref={plan.get('base_ref') or ''}\n") | |
| with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as fh: | |
| fh.write("## Eval plan\n\n") | |
| fh.write(f"- Base ref: `{plan.get('base_ref') or 'n/a'}`\n") | |
| fh.write(f"- Skills: {plan['skill_count']}\n") | |
| fh.write(f"- Scenarios: {plan['scenario_count']}\n") | |
| fh.write(f"- Runs: {plan['run_count']}\n") | |
| if plan["skills"]: | |
| fh.write(f"- Selected: `{', '.join(plan['skills'])}`\n") | |
| else: | |
| fh.write("- Selected: none\n") | |
| PY | |
| - name: Note fork PR limitation | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository | |
| run: | | |
| { | |
| echo | |
| echo "Model-backed smoke evals are skipped for fork PRs because repository secrets are not exposed to untrusted pull_request runs." | |
| echo "Maintainers can run the workflow manually with workflow_dispatch after reviewing the branch." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: eval-plan | |
| path: eval-plan.json | |
| smoke: | |
| name: Run smoke evals | |
| runs-on: ubuntu-latest | |
| needs: plan | |
| if: needs.plan.outputs.run_count != '0' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) | |
| timeout-minutes: 45 | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| steps: | |
| - name: Check ANTHROPIC_API_KEY availability | |
| id: key | |
| run: | | |
| if [ -z "$ANTHROPIC_API_KEY" ]; then | |
| echo "available=false" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "Model-backed smoke evals skipped: the ANTHROPIC_API_KEY repository secret is not configured." | |
| echo "Configure it to enable smoke evals on internal PRs, or run evals locally (evals/run_evals.py)." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "available=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/checkout@v4 | |
| if: steps.key.outputs.available == 'true' | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| if: steps.key.outputs.available == 'true' | |
| with: | |
| python-version: "3.12" | |
| - name: Install Python dependency | |
| if: steps.key.outputs.available == 'true' | |
| run: python -m pip install --upgrade pip pyyaml | |
| - name: Fetch base branch | |
| if: github.event_name == 'pull_request' && steps.key.outputs.available == 'true' | |
| run: git fetch origin "${{ github.base_ref }}" --depth=1 | |
| - name: Restore eval cache | |
| if: steps.key.outputs.available == 'true' | |
| id: cache-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: evals-workspace/cache | |
| key: evals-cache-${{ runner.os }}-${{ github.repository }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| restore-keys: | | |
| evals-cache-${{ runner.os }}-${{ github.repository }}-${{ github.base_ref }}- | |
| evals-cache-${{ runner.os }}-${{ github.repository }}-${{ github.ref_name }}- | |
| evals-cache-${{ runner.os }}-${{ github.repository }}- | |
| - name: Install claude CLI | |
| if: steps.key.outputs.available == 'true' | |
| run: | | |
| curl -fsSL https://claude.ai/install.sh | bash | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Verify claude CLI | |
| if: steps.key.outputs.available == 'true' | |
| run: | | |
| export PATH="$HOME/.local/bin:$PATH" | |
| claude --version | |
| - name: Run smoke evals | |
| if: steps.key.outputs.available == 'true' | |
| run: | | |
| export PATH="$HOME/.local/bin:$PATH" | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| BASE_REF="origin/${{ github.base_ref }}" | |
| else | |
| BASE_REF="origin/master" | |
| fi | |
| python evals/run_evals.py \ | |
| --changed-only \ | |
| --base-ref "$BASE_REF" \ | |
| --reuse-cache \ | |
| --selection-json eval-plan.json \ | |
| --iteration "ci-${{ github.run_id }}-${{ github.run_attempt }}" \ | |
| --workers 6 | |
| - name: Write benchmark summary | |
| if: always() && steps.key.outputs.available == 'true' | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| summary = Path(os.environ["GITHUB_STEP_SUMMARY"]) | |
| benchmark_files = sorted(Path("evals-workspace").glob("iteration-ci-*/benchmark.json")) | |
| if not benchmark_files: | |
| with summary.open("a") as fh: | |
| fh.write("\nNo benchmark artifact was produced.\n") | |
| raise SystemExit(0) | |
| benchmark = json.loads(benchmark_files[-1].read_text()) | |
| agg = benchmark["aggregate"] | |
| with summary.open("a") as fh: | |
| fh.write("\n## Smoke benchmark\n\n") | |
| fh.write(f"- With skill: {agg['with_skill']['mean_pass_rate']:.0%}\n") | |
| fh.write(f"- Without skill: {agg['without_skill']['mean_pass_rate']:.0%}\n") | |
| fh.write(f"- Delta: {agg['delta']:+.0%}\n") | |
| fh.write(f"- Cost (with skill): ${agg['with_skill']['total_cost_usd']:.2f}\n") | |
| fh.write(f"- Cost (without skill): ${agg['without_skill']['total_cost_usd']:.2f}\n") | |
| PY | |
| - name: Save eval cache | |
| if: always() && steps.key.outputs.available == 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: evals-workspace/cache | |
| key: ${{ steps.cache-restore.outputs.cache-primary-key }} | |
| - uses: actions/upload-artifact@v4 | |
| if: always() && steps.key.outputs.available == 'true' | |
| with: | |
| name: eval-benchmark | |
| path: | | |
| eval-plan.json | |
| evals-workspace/iteration-ci-*/benchmark.json |