From 722b17f184fd48e7ad5eb55fbdb3e808a1e60cdd Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 15:18:20 -0500 Subject: [PATCH 01/17] ci(evals): Run the AI evals on a labeled PR's docs preview An ai-evals:[:] label runs the sentry-for-ai-evals suites with sentry-for-ai's main skills on both sides. The head reads docs.sentry.io from the PR's Vercel preview and the base from the live docs, so the comparison shows what the PR's docs change. The plan job waits for the preview before it creates the runs on evals.sentry.dev, and fails with a clear message when Vercel builds no sentry-docs preview, as for a PR that only changes develop-docs. The live docs follow master, so a head that is behind master is missing what master gained since. The workflow's PR comment says how many commits behind the head was; updating the branch reruns the evals against master plus only the PR. The labels go in labels.yml, since the label sync deletes any label missing from it. Fixes VDY-307 Co-Authored-By: Claude Opus 5.5 --- .github/labels.yml | 8 + .github/workflows/ai-evals.yml | 282 +++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 .github/workflows/ai-evals.yml diff --git a/.github/labels.yml b/.github/labels.yml index a5e3ec38bc1804..229e38a64fefcb 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -284,6 +284,14 @@ color: 0E8A16 description: Prevents automatic closure of stale PRs +# AI evals - .github/workflows/ai-evals.yml +- name: 'ai-evals:products' + color: 6F42C1 + description: Run the products evals on every model, on this PR's preview and the live docs +- name: 'ai-evals:products:opus-5.5' + color: 6F42C1 + description: Run the products evals on Opus 5.5 only, on this PR's preview and the live docs + # Miscellaneous - name: API Docs color: 649B42 diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml new file mode 100644 index 00000000000000..7ae9530d5efbd1 --- /dev/null +++ b/.github/workflows/ai-evals.yml @@ -0,0 +1,282 @@ +# Runs the getsentry/sentry-for-ai-evals suites against this PR's docs. +# +# Label a PR `ai-evals:` to run the tests with that pytest mark, e.g. +# `ai-evals:products`, on each target (a harness with a model) that repo's plan-evals +# action lists, or `ai-evals::` to run them on the targets whose +# `:` contains ``, e.g. `ai-evals:products:opus-5.5`. The labels +# are in .github/labels.yml, since sync-labels.yml deletes any label that isn't. Each label +# runs the same skills, sentry-for-ai's `main`, with two sets of docs: the head reads +# docs.sentry.io from the PR's Vercel preview and the base from the live docs. The plan +# job creates one run per label and side on evals.sentry.dev, and each target's job sends +# its results into it. A new push reruns every eval label still on the PR. +# +# The base can't be the PR's merge base, since old production deploy URLs redirect to +# docs.sentry.io, so it's master as it is when the runs start. A head that's behind master +# is missing what master gained since, and the comparison counts that as the PR's. The PR +# comment says how far behind the head was, and updating the branch, which reruns the +# evals, makes the comparison show only the PR. +# +# sentry-for-ai-evals is internal, so the jobs check it out with the readonly bot's token +# and use its actions from that path. Fork PRs get no secrets, so they're skipped. This +# repo is public, so the eval outputs aren't kept as artifacts; the results go to +# evals.sentry.dev. + +name: AI Evals + +on: + pull_request: + types: [labeled, synchronize] + +permissions: {} + +jobs: + plan: + if: >- + github.event.pull_request.head.repo.full_name == github.repository && ( + (github.event.action == 'labeled' && startsWith(github.event.label.name, 'ai-evals:')) || + (github.event.action == 'synchronize' && contains(join(github.event.pull_request.labels.*.name, ','), 'ai-evals:')) + ) + runs-on: ubuntu-latest + # Mostly waiting for the preview, which is ready about 7 minutes after a push. + timeout-minutes: 30 + permissions: + contents: read + deployments: read + pull-requests: read + statuses: read + outputs: + runs: ${{ steps.plan.outputs.runs }} + count: ${{ steps.plan.outputs.count }} + jobs: ${{ steps.plan.outputs.jobs }} + evals-sha: ${{ steps.evals.outputs.sha }} + skills-sha: ${{ steps.skills.outputs.sha }} + docs-url: ${{ steps.preview.outputs.url }} + base-sha: ${{ steps.base.outputs.sha }} + behind-by: ${{ steps.base.outputs.behind-by }} + eval-runs: ${{ steps.create.outputs.runs }} + steps: + - name: Get sentry-for-ai-evals token + id: token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.SENTRY_READONLY_BOT_CLIENT_ID }} + private-key: ${{ secrets.SENTRY_READONLYBOT_PRIVATE_KEY }} + repositories: sentry-for-ai-evals + permission-contents: read + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: getsentry/sentry-for-ai-evals + token: ${{ steps.token.outputs.token }} + path: sentry-for-ai-evals + persist-credentials: false + + - id: plan + uses: ./sentry-for-ai-evals/.github/actions/plan-evals + + - id: evals + run: echo "sha=$(git -C sentry-for-ai-evals rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + # Both sides build the skills from the same commit, so only the docs differ. + - id: skills + run: | + ref="$(git ls-remote --exit-code https://github.com/getsentry/sentry-for-ai refs/heads/main)" + echo "sha=${ref%%$'\t'*}" >> "$GITHUB_OUTPUT" + + # vercel[bot] sets the commit status while the preview builds and creates the + # deployment, whose status has the URL, once it's done. This waits before the runs + # are created, since evals.sentry.dev fails a run that gets no result for 30 minutes. + - name: Wait for the Vercel preview + id: preview + if: steps.plan.outputs.count != '0' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + context="Vercel – sentry-docs" + environment="Preview – sentry-docs" + for (( i = 0; i < 40; i++ )); do + (( i == 0 )) || sleep 30 + # A newer push has a run of its own, so this one stops without creating runs. + current="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq .head.sha)" || continue + if [[ "$current" != "$HEAD_SHA" ]]; then + echo "::notice::The PR's head is now $current, which its own run evaluates" + exit 0 + fi + statuses="$(gh api "repos/$GITHUB_REPOSITORY/commits/$HEAD_SHA/status")" || continue + IFS=$'\t' read -r state description < <(jq -r --arg c "$context" \ + '.statuses[] | select(.context == $c) | [.state, .description] | @tsv' <<<"$statuses") || true + case "$state" in + failure | error) + echo "::error::The sentry-docs preview of $HEAD_SHA failed: $description" + exit 1 + ;; + success) + ids="$(gh api -X GET "repos/$GITHUB_REPOSITORY/deployments" \ + -f sha="$HEAD_SHA" -f environment="$environment" --jq '.[].id')" || continue + url="" + for id in $ids; do + url="$(gh api "repos/$GITHUB_REPOSITORY/deployments/$id/statuses" \ + --jq 'map(select(.state == "success"))[0].environment_url // empty')" || continue + [[ -n "$url" ]] && break + done + if [[ -n "$url" ]]; then + # The agent reads the .md exports, without a bypass token. + if ! curl -fsS -o /dev/null "$url/index.md"; then + echo "::error::The preview at $url doesn't serve index.md, so the agent couldn't read docs from it" + exit 1 + fi + echo "Preview: $url" + echo "url=$url" >> "$GITHUB_OUTPUT" + exit 0 + fi + # A PR that only changes develop-docs doesn't build sentry-docs. + if [[ "$description" == *"Ignored Build Step"* ]]; then + echo "::error::Vercel didn't build a sentry-docs preview of $HEAD_SHA ($description). A PR that only changes develop-docs has no sentry-docs preview to evaluate." + exit 1 + fi + ;; + esac + done + echo "::error::No sentry-docs preview of $HEAD_SHA after 20 minutes" + exit 1 + + # Right before the runs start, so the base is the live docs they'll read. behind_by + # counts the master commits the head is missing. + - id: base + if: steps.preview.outputs.url != '' + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + compare="$(gh api "repos/$GITHUB_REPOSITORY/compare/master...$HEAD_SHA?per_page=1" \ + --jq '"\(.base_commit.sha) \(.behind_by)"')" + read -r sha behind <<<"$compare" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "behind-by=$behind" >> "$GITHUB_OUTPUT" + + # Each run records this repo's commits, the head and master, and evals.sentry.dev + # pairs base and head runs by repository, so they never pair with sentry-for-ai's. + - id: create + if: steps.preview.outputs.url != '' + uses: ./sentry-for-ai-evals/.github/actions/create-runs + with: + runs: ${{ steps.plan.outputs.runs }} + skills-sha: ${{ steps.skills.outputs.sha }} + base-sha: ${{ steps.base.outputs.sha }} + evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} + + # Its own job, so the plan job's token can't write to the PR. + comment: + needs: plan + if: needs.plan.outputs.docs-url != '' + runs-on: ubuntu-latest + timeout-minutes: 5 + # Two plans finishing together would otherwise both create the comment. + concurrency: + group: ai-evals-comment-${{ github.event.pull_request.number }} + permissions: + pull-requests: write + steps: + - name: Comment on the PR + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + DOCS_URL: ${{ needs.plan.outputs.docs-url }} + BASE_SHA: ${{ needs.plan.outputs.base-sha }} + BEHIND_BY: ${{ needs.plan.outputs.behind-by }} + SKILLS_SHA: ${{ needs.plan.outputs.skills-sha }} + EVAL_RUNS: ${{ needs.plan.outputs.eval-runs }} + run: | + marker='' + commit() { echo "[\`${2:0:7}\`](https://github.com/$1/commit/$2)"; } + if (( BEHIND_BY == 0 )); then + drift="The head is up to date with \`master\`, so the comparison shows only this PR's changes." + else + (( BEHIND_BY == 1 )) && commits=commit || commits=commits + drift="The head is $BEHIND_BY $commits behind \`master\`, so the preview is missing changes the live docs have, and the comparison counts their removal as part of this PR. To compare only this PR's changes, update the branch, which reruns the evals." + fi + body="$(cat <-" to each run's {"id", "url"}, empty without an evals key. + runs="$(jq -r 'to_entries[] | "- [\(.key | sub("-(?base|head)$"; " (\(.side))"))](\(.value.url))"' <<<"$EVAL_RUNS")" + [[ -z "$runs" ]] || body+=$'\n\n'"Runs on evals.sentry.dev:"$'\n\n'"$runs" + + id="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | head -n 1)" + if [[ -n "$id" ]]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$id" -f body="$body" > /dev/null + else + gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" -f body="$body" > /dev/null + fi + + eval: + needs: plan + if: needs.plan.outputs.docs-url != '' + name: ${{ matrix.run.label }} (${{ matrix.revision }}, ${{ matrix.run.target.name }}) + # Per PR, label, revision and target, so the jobs don't cancel each other. + concurrency: + group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.run.label }}-${{ matrix.revision }}-${{ matrix.run.target.name }} + cancel-in-progress: true + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + # Each label's run on each target it picked. + run: ${{ fromJSON(needs.plan.outputs.jobs) }} + revision: [base, head] + steps: + - name: Get sentry-for-ai-evals token + id: token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.SENTRY_READONLY_BOT_CLIENT_ID }} + private-key: ${{ secrets.SENTRY_READONLYBOT_PRIVATE_KEY }} + repositories: sentry-for-ai-evals + permission-contents: read + + # Side by side, so the evals don't copy sentry-for-ai into their image. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: getsentry/sentry-for-ai-evals + ref: ${{ needs.plan.outputs.evals-sha }} + token: ${{ steps.token.outputs.token }} + path: sentry-for-ai-evals + persist-credentials: false + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: getsentry/sentry-for-ai + ref: ${{ needs.plan.outputs.skills-sha }} + path: sentry-for-ai + persist-credentials: false + + # Sends this target's results into the run the plan job created for the label and + # side. Only the head reads the preview. No artifact-name: this repo is public. + - uses: ./sentry-for-ai-evals/.github/actions/run-evals + with: + marker: ${{ matrix.run.marker }} + harness: ${{ matrix.run.target.harness }} + model: ${{ matrix.run.target.model }} + revision: ${{ matrix.revision }} + base-sha: ${{ needs.plan.outputs.base-sha }} + sentry-for-ai-path: sentry-for-ai + docs-url: ${{ matrix.revision == 'head' && needs.plan.outputs.docs-url || '' }} + run-id: ${{ fromJSON(needs.plan.outputs.eval-runs)[format('{0}-{1}', matrix.run.label, matrix.revision)].id }} + openrouter-api-key: ${{ secrets.AI_EVALS_OPENROUTER_API_KEY }} + evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} From df5da46e0a33bc72734b244f364561edfd06a9db Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 15:35:05 -0500 Subject: [PATCH 02/17] ci(evals): Add a Next.js-only eval label ai-evals:nextjs:opus-5.5 runs only the Next.js products test, once getsentry/sentry-for-ai-evals#88 gives it a nextjs mark. Refs VDY-307 Co-Authored-By: Claude Opus 5.5 --- .github/labels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index 229e38a64fefcb..00fa251ea6d665 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -291,6 +291,9 @@ - name: 'ai-evals:products:opus-5.5' color: 6F42C1 description: Run the products evals on Opus 5.5 only, on this PR's preview and the live docs +- name: 'ai-evals:nextjs:opus-5.5' + color: 6F42C1 + description: Run the Next.js products eval on Opus 5.5 only, on this PR's preview and the live docs # Miscellaneous - name: API Docs From 69a69f794640cf3e59434b8a4a105cceeb57dd51 Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 15:36:30 -0500 Subject: [PATCH 03/17] meta(labels): Keep the Good First Issue and js-v11-docs labels Both were created on the repo without an entry here, so the next label sync would delete them. Good First Issue is on open issues. Co-Authored-By: Claude Opus 5.5 --- .github/labels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index 00fa251ea6d665..51c670b3712c98 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -315,6 +315,8 @@ color: '584774' - name: gatsby-bug color: e99695 +- name: Good First Issue + color: 4cb782 - name: "Hacktoberfest \U0001F383" color: ffa500 - name: Hacktoberfest-accepted @@ -327,6 +329,8 @@ - name: javascript color: F6F6F8 description: Pull requests that update Javascript code +- name: js-v11-docs + color: '206131' - name: Mobile color: f2c94c - name: performance From 5e00cba4e0ab10cb07a1abff7fa3502f1ca123ec Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 15:53:42 -0500 Subject: [PATCH 04/17] ci(evals): Swap the Next.js eval label for the no-tools one ai-evals:no_tools:opus-5.5 runs the tests that give the agent no Sentry skills or MCP, only the docs, so base and head differ only in the docs. They come from getsentry/sentry-for-ai-evals#79 and need getsentry/sentry-for-ai-evals#75 to get runs on evals.sentry.dev. The PR comment no longer says the skills read the docs, since these tests install none. Refs VDY-307 Co-Authored-By: Claude Opus 5.5 --- .github/labels.yml | 4 ++-- .github/workflows/ai-evals.yml | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 51c670b3712c98..31c1b38f192fde 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -291,9 +291,9 @@ - name: 'ai-evals:products:opus-5.5' color: 6F42C1 description: Run the products evals on Opus 5.5 only, on this PR's preview and the live docs -- name: 'ai-evals:nextjs:opus-5.5' +- name: 'ai-evals:no_tools:opus-5.5' color: 6F42C1 - description: Run the Next.js products eval on Opus 5.5 only, on this PR's preview and the live docs + description: Run the evals without Sentry skills or MCP on Opus 5.5, on this PR's preview and live docs # Miscellaneous - name: API Docs diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 7ae9530d5efbd1..597c87a2e54301 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -3,12 +3,14 @@ # Label a PR `ai-evals:` to run the tests with that pytest mark, e.g. # `ai-evals:products`, on each target (a harness with a model) that repo's plan-evals # action lists, or `ai-evals::` to run them on the targets whose -# `:` contains ``, e.g. `ai-evals:products:opus-5.5`. The labels -# are in .github/labels.yml, since sync-labels.yml deletes any label that isn't. Each label -# runs the same skills, sentry-for-ai's `main`, with two sets of docs: the head reads -# docs.sentry.io from the PR's Vercel preview and the base from the live docs. The plan -# job creates one run per label and side on evals.sentry.dev, and each target's job sends -# its results into it. A new push reruns every eval label still on the PR. +# `:` contains ``, e.g. `ai-evals:products:opus-5.5`, or +# `ai-evals:no_tools:opus-5.5` for the tests that give the agent no Sentry skills or MCP, +# only the docs. The labels are in .github/labels.yml, since sync-labels.yml deletes any +# label that isn't. Each label runs with two sets of docs and the same skills, from +# sentry-for-ai's `main`: the head reads docs.sentry.io from the PR's Vercel preview and +# the base from the live docs. The plan job creates one run per label and side on +# evals.sentry.dev, and each target's job sends its results into it. A new push reruns +# every eval label still on the PR. # # The base can't be the PR's merge base, since old production deploy URLs redirect to # docs.sentry.io, so it's master as it is when the runs start. A head that's behind master @@ -202,11 +204,13 @@ jobs: $marker ### AI evals - The skills from sentry-for-ai's \`main\` at $(commit getsentry/sentry-for-ai "$SKILLS_SHA") read docs.sentry.io from: + Base and head differ only in where docs.sentry.io comes from: - **Head:** this PR's preview of $(commit "$GITHUB_REPOSITORY" "$HEAD_SHA"), $DOCS_URL - **Base:** the live docs, \`master\` at $(commit "$GITHUB_REPOSITORY" "$BASE_SHA") + Tests that install the Sentry skills build them from sentry-for-ai's \`main\` at $(commit getsentry/sentry-for-ai "$SKILLS_SHA") on both sides. + $drift EOF )" From 1fb6b1b6a299ccb697c9dde9ed2ce71368a5c068 Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 16:02:17 -0500 Subject: [PATCH 05/17] ci(evals): Add an llms.txt-only eval label ai-evals:llms_txt:opus-5.5 runs only the no-tools test that starts from docs.sentry.io/llms.txt, for PRs that change llms.txt. It needs the llms_txt mark from getsentry/sentry-for-ai-evals#89. Refs VDY-307 Co-Authored-By: Claude Opus 5.5 --- .github/labels.yml | 3 +++ .github/workflows/ai-evals.yml | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 31c1b38f192fde..113024a7315b88 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -294,6 +294,9 @@ - name: 'ai-evals:no_tools:opus-5.5' color: 6F42C1 description: Run the evals without Sentry skills or MCP on Opus 5.5, on this PR's preview and live docs +- name: 'ai-evals:llms_txt:opus-5.5' + color: 6F42C1 + description: Run the llms.txt eval on Opus 5.5, on this PR's preview and the live docs # Miscellaneous - name: API Docs diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 597c87a2e54301..e014c1f0079b68 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -5,8 +5,9 @@ # action lists, or `ai-evals::` to run them on the targets whose # `:` contains ``, e.g. `ai-evals:products:opus-5.5`, or # `ai-evals:no_tools:opus-5.5` for the tests that give the agent no Sentry skills or MCP, -# only the docs. The labels are in .github/labels.yml, since sync-labels.yml deletes any -# label that isn't. Each label runs with two sets of docs and the same skills, from +# only the docs, and `ai-evals:llms_txt:opus-5.5` for the one of those that starts from +# llms.txt. The labels are in .github/labels.yml, since sync-labels.yml deletes any label +# that isn't. Each label runs with two sets of docs and the same skills, from # sentry-for-ai's `main`: the head reads docs.sentry.io from the PR's Vercel preview and # the base from the live docs. The plan job creates one run per label and side on # evals.sentry.dev, and each target's job sends its results into it. A new push reruns From ef02fd9673d9b864621d034dab39a531739c6c51 Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 16:46:57 -0500 Subject: [PATCH 06/17] ci(evals): Run each eval test in its own job A label's tests ran one after another in one container per target and side, so the llms.txt test waited behind the other no-tools openings and each saw what the one before it left behind. sentry-for-ai-evals' create-runs now lists a job per test, target and side, and the matrix runs those. --- .github/workflows/ai-evals.yml | 35 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index e014c1f0079b68..5cebb6d430503f 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -10,8 +10,9 @@ # that isn't. Each label runs with two sets of docs and the same skills, from # sentry-for-ai's `main`: the head reads docs.sentry.io from the PR's Vercel preview and # the base from the live docs. The plan job creates one run per label and side on -# evals.sentry.dev, and each target's job sends its results into it. A new push reruns -# every eval label still on the PR. +# evals.sentry.dev, and a job per test, target and side sends that test's result into +# it, each test in a clean container and all of them at once. A new push reruns every +# eval label still on the PR. # # The base can't be the PR's merge base, since old production deploy URLs redirect to # docs.sentry.io, so it's master as it is when the runs start. A head that's behind master @@ -50,13 +51,13 @@ jobs: outputs: runs: ${{ steps.plan.outputs.runs }} count: ${{ steps.plan.outputs.count }} - jobs: ${{ steps.plan.outputs.jobs }} evals-sha: ${{ steps.evals.outputs.sha }} skills-sha: ${{ steps.skills.outputs.sha }} docs-url: ${{ steps.preview.outputs.url }} base-sha: ${{ steps.base.outputs.sha }} behind-by: ${{ steps.base.outputs.behind-by }} eval-runs: ${{ steps.create.outputs.runs }} + eval-jobs: ${{ steps.create.outputs.jobs }} steps: - name: Get sentry-for-ai-evals token id: token @@ -229,11 +230,11 @@ jobs: eval: needs: plan - if: needs.plan.outputs.docs-url != '' - name: ${{ matrix.run.label }} (${{ matrix.revision }}, ${{ matrix.run.target.name }}) - # Per PR, label, revision and target, so the jobs don't cancel each other. + if: needs.plan.outputs.docs-url != '' && needs.plan.outputs.eval-jobs != '[]' + name: ${{ matrix.job.label }} (${{ matrix.job.revision }}, ${{ matrix.job.target.name }}, ${{ matrix.job.test-name }}) + # Per PR, label, revision, target and test, so the jobs don't cancel each other. concurrency: - group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.run.label }}-${{ matrix.revision }}-${{ matrix.run.target.name }} + group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.job.label }}-${{ matrix.job.revision }}-${{ matrix.job.target.name }}-${{ matrix.job.test }} cancel-in-progress: true runs-on: ubuntu-latest timeout-minutes: 60 @@ -242,9 +243,8 @@ jobs: strategy: fail-fast: false matrix: - # Each label's run on each target it picked. - run: ${{ fromJSON(needs.plan.outputs.jobs) }} - revision: [base, head] + # Each test a label selects, on each target the label picked, at each side. + job: ${{ fromJSON(needs.plan.outputs.eval-jobs) }} steps: - name: Get sentry-for-ai-evals token id: token @@ -271,17 +271,18 @@ jobs: path: sentry-for-ai persist-credentials: false - # Sends this target's results into the run the plan job created for the label and + # Sends this test's result into the run the plan job created for the label and # side. Only the head reads the preview. No artifact-name: this repo is public. - uses: ./sentry-for-ai-evals/.github/actions/run-evals with: - marker: ${{ matrix.run.marker }} - harness: ${{ matrix.run.target.harness }} - model: ${{ matrix.run.target.model }} - revision: ${{ matrix.revision }} + marker: ${{ matrix.job.marker }} + test: ${{ matrix.job.test }} + harness: ${{ matrix.job.target.harness }} + model: ${{ matrix.job.target.model }} + revision: ${{ matrix.job.revision }} base-sha: ${{ needs.plan.outputs.base-sha }} sentry-for-ai-path: sentry-for-ai - docs-url: ${{ matrix.revision == 'head' && needs.plan.outputs.docs-url || '' }} - run-id: ${{ fromJSON(needs.plan.outputs.eval-runs)[format('{0}-{1}', matrix.run.label, matrix.revision)].id }} + docs-url: ${{ matrix.job.revision == 'head' && needs.plan.outputs.docs-url || '' }} + run-id: ${{ fromJSON(needs.plan.outputs.eval-runs)[format('{0}-{1}', matrix.job.label, matrix.job.revision)].id }} openrouter-api-key: ${{ secrets.AI_EVALS_OPENROUTER_API_KEY }} evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} From 1ee05e236a7e6d86f891b9eb2ebfe120d569a3b4 Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 17:27:21 -0500 Subject: [PATCH 07/17] ci(evals): Keep one llms.txt eval label, on Sonnet Drops the Opus 5.5 labels for the products, no-tools and llms.txt evals and adds `ai-evals:llms_txt:sonnet-5.5`, so the eval labels are `ai-evals:products` and that one. --- .github/labels.yml | 10 ++-------- .github/workflows/ai-evals.yml | 7 +++---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 113024a7315b88..58020131daa01d 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -288,15 +288,9 @@ - name: 'ai-evals:products' color: 6F42C1 description: Run the products evals on every model, on this PR's preview and the live docs -- name: 'ai-evals:products:opus-5.5' +- name: 'ai-evals:llms_txt:sonnet-5.5' color: 6F42C1 - description: Run the products evals on Opus 5.5 only, on this PR's preview and the live docs -- name: 'ai-evals:no_tools:opus-5.5' - color: 6F42C1 - description: Run the evals without Sentry skills or MCP on Opus 5.5, on this PR's preview and live docs -- name: 'ai-evals:llms_txt:opus-5.5' - color: 6F42C1 - description: Run the llms.txt eval on Opus 5.5, on this PR's preview and the live docs + description: Run the llms.txt eval on Sonnet 5.5, on this PR's preview and the live docs # Miscellaneous - name: API Docs diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 5cebb6d430503f..54d249c6a6be27 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -3,10 +3,9 @@ # Label a PR `ai-evals:` to run the tests with that pytest mark, e.g. # `ai-evals:products`, on each target (a harness with a model) that repo's plan-evals # action lists, or `ai-evals::` to run them on the targets whose -# `:` contains ``, e.g. `ai-evals:products:opus-5.5`, or -# `ai-evals:no_tools:opus-5.5` for the tests that give the agent no Sentry skills or MCP, -# only the docs, and `ai-evals:llms_txt:opus-5.5` for the one of those that starts from -# llms.txt. The labels are in .github/labels.yml, since sync-labels.yml deletes any label +# `:` contains ``, e.g. `ai-evals:llms_txt:sonnet-5.5` for the +# test that starts from llms.txt with no Sentry skills or MCP, only the docs, on Sonnet +# 5.5. The labels are in .github/labels.yml, since sync-labels.yml deletes any label # that isn't. Each label runs with two sets of docs and the same skills, from # sentry-for-ai's `main`: the head reads docs.sentry.io from the PR's Vercel preview and # the base from the live docs. The plan job creates one run per label and side on From 869e294c9c561c57a4467474062c053fa943d4b6 Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 17:38:13 -0500 Subject: [PATCH 08/17] ci(evals): Run the llms.txt eval label 3 times per side sentry-for-ai-evals labels can end in `:x` to repeat each test, so the llms.txt label becomes `ai-evals:llms_txt:sonnet-5.5:x3`, and a docs change gets three samples per side instead of one. --- .github/labels.yml | 4 ++-- .github/workflows/ai-evals.yml | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 58020131daa01d..28fbf106f00316 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -288,9 +288,9 @@ - name: 'ai-evals:products' color: 6F42C1 description: Run the products evals on every model, on this PR's preview and the live docs -- name: 'ai-evals:llms_txt:sonnet-5.5' +- name: 'ai-evals:llms_txt:sonnet-5.5:x3' color: 6F42C1 - description: Run the llms.txt eval on Sonnet 5.5, on this PR's preview and the live docs + description: Run the llms.txt eval 3 times on Sonnet 5.5, on this PR's preview and the live docs # Miscellaneous - name: API Docs diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 54d249c6a6be27..09b3121f6fa0d3 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -3,15 +3,15 @@ # Label a PR `ai-evals:` to run the tests with that pytest mark, e.g. # `ai-evals:products`, on each target (a harness with a model) that repo's plan-evals # action lists, or `ai-evals::` to run them on the targets whose -# `:` contains ``, e.g. `ai-evals:llms_txt:sonnet-5.5` for the -# test that starts from llms.txt with no Sentry skills or MCP, only the docs, on Sonnet -# 5.5. The labels are in .github/labels.yml, since sync-labels.yml deletes any label -# that isn't. Each label runs with two sets of docs and the same skills, from -# sentry-for-ai's `main`: the head reads docs.sentry.io from the PR's Vercel preview and -# the base from the live docs. The plan job creates one run per label and side on -# evals.sentry.dev, and a job per test, target and side sends that test's result into -# it, each test in a clean container and all of them at once. A new push reruns every -# eval label still on the PR. +# `:` contains ``, and a last `:x` runs each test that many +# times, e.g. `ai-evals:llms_txt:sonnet-5.5:x3` for three runs on Sonnet 5.5 of the test +# that starts from llms.txt with no Sentry skills or MCP, only the docs. The labels are in +# .github/labels.yml, since sync-labels.yml deletes any label that isn't. Each label runs +# with two sets of docs and the same skills, from sentry-for-ai's `main`: the head reads +# docs.sentry.io from the PR's Vercel preview and the base from the live docs. The plan +# job creates one run per label and side on evals.sentry.dev, and a job per test, target +# and side sends that test's result into it, each test in a clean container and all of +# them at once. A new push reruns every eval label still on the PR. # # The base can't be the PR's merge base, since old production deploy URLs redirect to # docs.sentry.io, so it's master as it is when the runs start. A head that's behind master From a869c8145e0bf5d22906993c9369fe9e08ff4f1b Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 18:25:45 -0500 Subject: [PATCH 09/17] ci(evals): Warn on a stale head instead of commenting on the PR sentry-evals[bot] already comments on the PR with each label's runs, so the workflow's own comment repeated them, and a newly added label replaced the earlier labels' links in it. The one thing only it said, that a head behind master counts master's changes as the PR's, is now a warning from the plan job, and the workflow no longer has a token that can write to the PR. --- .github/workflows/ai-evals.yml | 68 +++------------------------------- 1 file changed, 6 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 09b3121f6fa0d3..9dab24a7dc5fc9 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -15,9 +15,9 @@ # # The base can't be the PR's merge base, since old production deploy URLs redirect to # docs.sentry.io, so it's master as it is when the runs start. A head that's behind master -# is missing what master gained since, and the comparison counts that as the PR's. The PR -# comment says how far behind the head was, and updating the branch, which reruns the -# evals, makes the comparison show only the PR. +# is missing what master gained since, and the comparison counts that as the PR's. The plan +# job warns when the head is behind, and updating the branch, which reruns the evals, makes +# the comparison show only the PR. sentry-evals[bot] comments on the PR with the runs. # # sentry-for-ai-evals is internal, so the jobs check it out with the readonly bot's token # and use its actions from that path. Fork PRs get no secrets, so they're skipped. This @@ -54,7 +54,6 @@ jobs: skills-sha: ${{ steps.skills.outputs.sha }} docs-url: ${{ steps.preview.outputs.url }} base-sha: ${{ steps.base.outputs.sha }} - behind-by: ${{ steps.base.outputs.behind-by }} eval-runs: ${{ steps.create.outputs.runs }} eval-jobs: ${{ steps.create.outputs.jobs }} steps: @@ -157,7 +156,9 @@ jobs: --jq '"\(.base_commit.sha) \(.behind_by)"')" read -r sha behind <<<"$compare" echo "sha=$sha" >> "$GITHUB_OUTPUT" - echo "behind-by=$behind" >> "$GITHUB_OUTPUT" + if (( behind > 0 )); then + echo "::warning::The head is missing $behind of master's commits, so the preview lacks changes the live docs have, and the comparison counts their removal as part of this PR. Update the branch to compare only this PR's changes, which reruns the evals." + fi # Each run records this repo's commits, the head and master, and evals.sentry.dev # pairs base and head runs by repository, so they never pair with sentry-for-ai's. @@ -170,63 +171,6 @@ jobs: base-sha: ${{ steps.base.outputs.sha }} evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} - # Its own job, so the plan job's token can't write to the PR. - comment: - needs: plan - if: needs.plan.outputs.docs-url != '' - runs-on: ubuntu-latest - timeout-minutes: 5 - # Two plans finishing together would otherwise both create the comment. - concurrency: - group: ai-evals-comment-${{ github.event.pull_request.number }} - permissions: - pull-requests: write - steps: - - name: Comment on the PR - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - DOCS_URL: ${{ needs.plan.outputs.docs-url }} - BASE_SHA: ${{ needs.plan.outputs.base-sha }} - BEHIND_BY: ${{ needs.plan.outputs.behind-by }} - SKILLS_SHA: ${{ needs.plan.outputs.skills-sha }} - EVAL_RUNS: ${{ needs.plan.outputs.eval-runs }} - run: | - marker='' - commit() { echo "[\`${2:0:7}\`](https://github.com/$1/commit/$2)"; } - if (( BEHIND_BY == 0 )); then - drift="The head is up to date with \`master\`, so the comparison shows only this PR's changes." - else - (( BEHIND_BY == 1 )) && commits=commit || commits=commits - drift="The head is $BEHIND_BY $commits behind \`master\`, so the preview is missing changes the live docs have, and the comparison counts their removal as part of this PR. To compare only this PR's changes, update the branch, which reruns the evals." - fi - body="$(cat <-" to each run's {"id", "url"}, empty without an evals key. - runs="$(jq -r 'to_entries[] | "- [\(.key | sub("-(?base|head)$"; " (\(.side))"))](\(.value.url))"' <<<"$EVAL_RUNS")" - [[ -z "$runs" ]] || body+=$'\n\n'"Runs on evals.sentry.dev:"$'\n\n'"$runs" - - id="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ - --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | head -n 1)" - if [[ -n "$id" ]]; then - gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$id" -f body="$body" > /dev/null - else - gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" -f body="$body" > /dev/null - fi - eval: needs: plan if: needs.plan.outputs.docs-url != '' && needs.plan.outputs.eval-jobs != '[]' From ee736850b424190737c5e58272e8d580c6c2c64e Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 18:34:13 -0500 Subject: [PATCH 10/17] ci(evals): Take the evals commit from its checkout step The plan job's `runs` and `count` outputs had no reader, since the eval job checks `docs-url`, and actions/checkout already outputs the commit it checked out, so the step that ran git rev-parse for it is gone. --- .github/workflows/ai-evals.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 9dab24a7dc5fc9..9065e259452d8b 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -48,9 +48,7 @@ jobs: pull-requests: read statuses: read outputs: - runs: ${{ steps.plan.outputs.runs }} - count: ${{ steps.plan.outputs.count }} - evals-sha: ${{ steps.evals.outputs.sha }} + evals-sha: ${{ steps.evals.outputs.commit }} skills-sha: ${{ steps.skills.outputs.sha }} docs-url: ${{ steps.preview.outputs.url }} base-sha: ${{ steps.base.outputs.sha }} @@ -66,7 +64,8 @@ jobs: repositories: sentry-for-ai-evals permission-contents: read - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - id: evals + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: getsentry/sentry-for-ai-evals token: ${{ steps.token.outputs.token }} @@ -76,9 +75,6 @@ jobs: - id: plan uses: ./sentry-for-ai-evals/.github/actions/plan-evals - - id: evals - run: echo "sha=$(git -C sentry-for-ai-evals rev-parse HEAD)" >> "$GITHUB_OUTPUT" - # Both sides build the skills from the same commit, so only the docs differ. - id: skills run: | From 2a24dbdbb1857551460a148c22b92c253eec45cf Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 19:01:48 -0500 Subject: [PATCH 11/17] ci(evals): Read the base docs from the merge base's production deploy The base read today's live docs, so a head behind master had the commits it was missing counted as part of the PR. The plan job now finds the merge base's production deploy and gives its URL to the base jobs, whose docs proxy sends the header that keeps it from redirecting to docs.sentry.io. A deploy built before middleware.ts checked for that header still redirects, so for those, or a merge base with no production deploy, the base reads the live docs and the plan job warns as before. Refs VDY-307 --- .github/workflows/ai-evals.yml | 63 ++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 9065e259452d8b..42ad9392d203f3 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -8,16 +8,20 @@ # that starts from llms.txt with no Sentry skills or MCP, only the docs. The labels are in # .github/labels.yml, since sync-labels.yml deletes any label that isn't. Each label runs # with two sets of docs and the same skills, from sentry-for-ai's `main`: the head reads -# docs.sentry.io from the PR's Vercel preview and the base from the live docs. The plan -# job creates one run per label and side on evals.sentry.dev, and a job per test, target -# and side sends that test's result into it, each test in a clean container and all of -# them at once. A new push reruns every eval label still on the PR. +# docs.sentry.io from the PR's Vercel preview and the base from the production deploy of +# the PR's merge base. The plan job creates one run per label and side on +# evals.sentry.dev, and a job per test, target and side sends that test's result into it, +# each test in a clean container and all of them at once. A new push reruns every eval +# label still on the PR. # -# The base can't be the PR's merge base, since old production deploy URLs redirect to -# docs.sentry.io, so it's master as it is when the runs start. A head that's behind master -# is missing what master gained since, and the comparison counts that as the PR's. The plan -# job warns when the head is behind, and updating the branch, which reruns the evals, makes -# the comparison show only the PR. sentry-evals[bot] comments on the PR with the runs. +# A production deploy redirects every page to docs.sentry.io unless the request has the +# x-sentry-evals-pin header that the evals' docs proxy sends (middleware.ts). Deploys built +# before middleware.ts checked for it redirect anyway, so a merge base like that, or one +# with no production deploy, gets the live docs as the base, which is master as it is when +# the runs start. A head that's behind master is then missing what master gained since, and +# the comparison counts that as the PR's. The plan job warns then, and updating the branch, +# which reruns the evals, makes the comparison show only the PR. sentry-evals[bot] comments +# on the PR with the runs. # # sentry-for-ai-evals is internal, so the jobs check it out with the readonly bot's token # and use its actions from that path. Fork PRs get no secrets, so they're skipped. This @@ -51,6 +55,7 @@ jobs: evals-sha: ${{ steps.evals.outputs.commit }} skills-sha: ${{ steps.skills.outputs.sha }} docs-url: ${{ steps.preview.outputs.url }} + base-docs-url: ${{ steps.base.outputs.url }} base-sha: ${{ steps.base.outputs.sha }} eval-runs: ${{ steps.create.outputs.runs }} eval-jobs: ${{ steps.create.outputs.jobs }} @@ -140,8 +145,10 @@ jobs: echo "::error::No sentry-docs preview of $HEAD_SHA after 20 minutes" exit 1 - # Right before the runs start, so the base is the live docs they'll read. behind_by - # counts the master commits the head is missing. + # The merge base's production deploy, if it serves its own pages to the pin header. + # curl doesn't follow redirects, so a deploy that redirects anyway isn't a 200. Else + # the live docs, right before the runs start so they're the ones the runs read, and + # behind_by counts the master commits the head is missing. - id: base if: steps.preview.outputs.url != '' env: @@ -149,15 +156,31 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | compare="$(gh api "repos/$GITHUB_REPOSITORY/compare/master...$HEAD_SHA?per_page=1" \ - --jq '"\(.base_commit.sha) \(.behind_by)"')" - read -r sha behind <<<"$compare" - echo "sha=$sha" >> "$GITHUB_OUTPUT" + --jq '"\(.merge_base_commit.sha) \(.base_commit.sha) \(.behind_by)"')" + read -r merge_base master behind <<<"$compare" + ids="$(gh api -X GET "repos/$GITHUB_REPOSITORY/deployments" \ + -f sha="$merge_base" -f environment="Production – sentry-docs" --jq '.[].id')" || ids="" + url="" + for id in $ids; do + url="$(gh api "repos/$GITHUB_REPOSITORY/deployments/$id/statuses" \ + --jq 'map(select(.state == "success"))[0].environment_url // empty')" || continue + [[ -n "$url" ]] && break + done + if [[ -n "$url" ]] && [[ "$(curl -sS -o /dev/null -w '%{http_code}' \ + -H 'x-sentry-evals-pin: 1' "$url/index.md")" == 200 ]]; then + echo "Base: $url, the production deploy of the merge base $merge_base" + echo "sha=$merge_base" >> "$GITHUB_OUTPUT" + echo "url=$url" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "sha=$master" >> "$GITHUB_OUTPUT" if (( behind > 0 )); then - echo "::warning::The head is missing $behind of master's commits, so the preview lacks changes the live docs have, and the comparison counts their removal as part of this PR. Update the branch to compare only this PR's changes, which reruns the evals." + echo "::warning::The merge base $merge_base has no production deploy that serves its own pages, so the base reads the live docs. The head is missing $behind of master's commits, so the preview lacks changes the live docs have, and the comparison counts their removal as part of this PR. Update the branch to compare only this PR's changes, which reruns the evals." fi - # Each run records this repo's commits, the head and master, and evals.sentry.dev - # pairs base and head runs by repository, so they never pair with sentry-for-ai's. + # Each run records this repo's commits, the head and the base's (the merge base or + # master), and evals.sentry.dev pairs base and head runs by repository, so they + # never pair with sentry-for-ai's. - id: create if: steps.preview.outputs.url != '' uses: ./sentry-for-ai-evals/.github/actions/create-runs @@ -211,7 +234,9 @@ jobs: persist-credentials: false # Sends this test's result into the run the plan job created for the label and - # side. Only the head reads the preview. No artifact-name: this repo is public. + # side. The head reads the preview, and the base the merge base's production deploy + # when the plan job found one, or else the live docs. No artifact-name: this repo is + # public. - uses: ./sentry-for-ai-evals/.github/actions/run-evals with: marker: ${{ matrix.job.marker }} @@ -221,7 +246,7 @@ jobs: revision: ${{ matrix.job.revision }} base-sha: ${{ needs.plan.outputs.base-sha }} sentry-for-ai-path: sentry-for-ai - docs-url: ${{ matrix.job.revision == 'head' && needs.plan.outputs.docs-url || '' }} + docs-url: ${{ matrix.job.revision == 'head' && needs.plan.outputs.docs-url || needs.plan.outputs.base-docs-url }} run-id: ${{ fromJSON(needs.plan.outputs.eval-runs)[format('{0}-{1}', matrix.job.label, matrix.job.revision)].id }} openrouter-api-key: ${{ secrets.AI_EVALS_OPENROUTER_API_KEY }} evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} From b3e433b8e6bfb86533246d87658aa474a0598a0d Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Fri, 23 Oct 2026 19:27:40 -0500 Subject: [PATCH 12/17] ci(evals): Compare the head against the PR's base sha, not master Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ai-evals.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 42ad9392d203f3..fd8db0c357ad75 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -148,16 +148,17 @@ jobs: # The merge base's production deploy, if it serves its own pages to the pin header. # curl doesn't follow redirects, so a deploy that redirects anyway isn't a 200. Else # the live docs, right before the runs start so they're the ones the runs read, and - # behind_by counts the master commits the head is missing. + # behind_by counts the base branch's commits the head is missing. - id: base if: steps.preview.outputs.url != '' env: GH_TOKEN: ${{ github.token }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - compare="$(gh api "repos/$GITHUB_REPOSITORY/compare/master...$HEAD_SHA?per_page=1" \ - --jq '"\(.merge_base_commit.sha) \(.base_commit.sha) \(.behind_by)"')" - read -r merge_base master behind <<<"$compare" + compare="$(gh api "repos/$GITHUB_REPOSITORY/compare/$BASE_SHA...$HEAD_SHA?per_page=1" \ + --jq '"\(.merge_base_commit.sha) \(.behind_by)"')" + read -r merge_base behind <<<"$compare" ids="$(gh api -X GET "repos/$GITHUB_REPOSITORY/deployments" \ -f sha="$merge_base" -f environment="Production – sentry-docs" --jq '.[].id')" || ids="" url="" @@ -173,14 +174,14 @@ jobs: echo "url=$url" >> "$GITHUB_OUTPUT" exit 0 fi - echo "sha=$master" >> "$GITHUB_OUTPUT" + echo "sha=$BASE_SHA" >> "$GITHUB_OUTPUT" if (( behind > 0 )); then - echo "::warning::The merge base $merge_base has no production deploy that serves its own pages, so the base reads the live docs. The head is missing $behind of master's commits, so the preview lacks changes the live docs have, and the comparison counts their removal as part of this PR. Update the branch to compare only this PR's changes, which reruns the evals." + echo "::warning::The merge base $merge_base has no production deploy that serves its own pages, so the base reads the live docs. The head is missing $behind of the base branch's commits, so the preview lacks changes the live docs have, and the comparison counts their removal as part of this PR. Update the branch to compare only this PR's changes, which reruns the evals." fi - # Each run records this repo's commits, the head and the base's (the merge base or - # master), and evals.sentry.dev pairs base and head runs by repository, so they - # never pair with sentry-for-ai's. + # Each run records this repo's commits, the head and the base's (the merge base or the + # base branch's tip), and evals.sentry.dev pairs base and head runs by repository, so + # they never pair with sentry-for-ai's. - id: create if: steps.preview.outputs.url != '' uses: ./sentry-for-ai-evals/.github/actions/create-runs From 70cce437287e18266754c0704dc0a47726723c52 Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Mon, 26 Oct 2026 10:33:43 -0500 Subject: [PATCH 13/17] ci(evals): Count commits behind the live base branch, not base.sha pull_request.base.sha is a snapshot from the event and can lag the base branch, so behind_by stayed 0 and the fallback never warned. Compare against base.ref and record the tip it resolves to. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ai-evals.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index fd8db0c357ad75..180add0af19ff6 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -153,12 +153,13 @@ jobs: if: steps.preview.outputs.url != '' env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + # The branch, not base.sha, which is a snapshot from the event and can lag it. + BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - compare="$(gh api "repos/$GITHUB_REPOSITORY/compare/$BASE_SHA...$HEAD_SHA?per_page=1" \ - --jq '"\(.merge_base_commit.sha) \(.behind_by)"')" - read -r merge_base behind <<<"$compare" + compare="$(gh api "repos/$GITHUB_REPOSITORY/compare/$BASE_REF...$HEAD_SHA?per_page=1" \ + --jq '"\(.merge_base_commit.sha) \(.base_commit.sha) \(.behind_by)"')" + read -r merge_base base_tip behind <<<"$compare" ids="$(gh api -X GET "repos/$GITHUB_REPOSITORY/deployments" \ -f sha="$merge_base" -f environment="Production – sentry-docs" --jq '.[].id')" || ids="" url="" @@ -174,7 +175,7 @@ jobs: echo "url=$url" >> "$GITHUB_OUTPUT" exit 0 fi - echo "sha=$BASE_SHA" >> "$GITHUB_OUTPUT" + echo "sha=$base_tip" >> "$GITHUB_OUTPUT" if (( behind > 0 )); then echo "::warning::The merge base $merge_base has no production deploy that serves its own pages, so the base reads the live docs. The head is missing $behind of the base branch's commits, so the preview lacks changes the live docs have, and the comparison counts their removal as part of this PR. Update the branch to compare only this PR's changes, which reruns the evals." fi From bee32b94ce0c4d54cb2f158d420e824534a03dc7 Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Mon, 26 Oct 2026 13:04:56 -0500 Subject: [PATCH 14/17] ci(evals): Name the eval jobs by test number, not test name sentry-for-ai-evals is internal, and this repo's checks and logs listed its test names: in each job's name, and in run-evals' `test:` input, which GitHub prints at the top of the step. The jobs now pass the test's number (`test-index`) and count, which run-evals turns back into the test, and are named `test `. evals.sentry.dev has each test's result. Needs getsentry/sentry-for-ai-evals#97. Refs VDY-312 --- .github/workflows/ai-evals.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 180add0af19ff6..48e8819c2cceff 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -195,10 +195,12 @@ jobs: eval: needs: plan if: needs.plan.outputs.docs-url != '' && needs.plan.outputs.eval-jobs != '[]' - name: ${{ matrix.job.label }} (${{ matrix.job.revision }}, ${{ matrix.job.target.name }}, ${{ matrix.job.test-name }}) + # By the test's number, not its name: sentry-for-ai-evals is internal, so this public + # repo's checks and logs don't name its tests. evals.sentry.dev has each test's result. + name: ${{ matrix.job.label }} (${{ matrix.job.revision }}, ${{ matrix.job.target.name }}, test ${{ matrix.job.test-index }}) # Per PR, label, revision, target and test, so the jobs don't cancel each other. concurrency: - group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.job.label }}-${{ matrix.job.revision }}-${{ matrix.job.target.name }}-${{ matrix.job.test }} + group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.job.label }}-${{ matrix.job.revision }}-${{ matrix.job.target.name }}-${{ matrix.job.test-index }} cancel-in-progress: true runs-on: ubuntu-latest timeout-minutes: 60 @@ -242,7 +244,8 @@ jobs: - uses: ./sentry-for-ai-evals/.github/actions/run-evals with: marker: ${{ matrix.job.marker }} - test: ${{ matrix.job.test }} + test-index: ${{ matrix.job.test-index }} + count: ${{ matrix.job.count }} harness: ${{ matrix.job.target.harness }} model: ${{ matrix.job.target.model }} revision: ${{ matrix.job.revision }} From a40ad2cdd4c63edf3a0c916b6b414b6d5da4426d Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Mon, 26 Oct 2026 13:51:11 -0500 Subject: [PATCH 15/17] ci(evals): Pass the job's test total to run-evals run-evals checks it against the tests it collects, so a job fails rather than run another test than create-runs numbered. Needs getsentry/sentry-for-ai-evals#97. Refs VDY-312 --- .github/workflows/ai-evals.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 48e8819c2cceff..7602f218c67755 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -245,6 +245,7 @@ jobs: with: marker: ${{ matrix.job.marker }} test-index: ${{ matrix.job.test-index }} + test-total: ${{ matrix.job.test-total }} count: ${{ matrix.job.count }} harness: ${{ matrix.job.target.harness }} model: ${{ matrix.job.target.model }} From 0d041f995a3e3a47fe7eb9cbcbfd4fcaa052e3ad Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Mon, 26 Oct 2026 14:26:18 -0500 Subject: [PATCH 16/17] ci(evals): Drop the products eval label ai-evals:products runs both products tests on every model, at both the head and the base, so 8 jobs each time it's added and again on every push. The llms.txt label on Sonnet stays, and with no products label in labels.yml the sync won't create one. --- .github/labels.yml | 3 --- .github/workflows/ai-evals.yml | 25 ++++++++++++------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 28fbf106f00316..a3eaab2c3df7ca 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -285,9 +285,6 @@ description: Prevents automatic closure of stale PRs # AI evals - .github/workflows/ai-evals.yml -- name: 'ai-evals:products' - color: 6F42C1 - description: Run the products evals on every model, on this PR's preview and the live docs - name: 'ai-evals:llms_txt:sonnet-5.5:x3' color: 6F42C1 description: Run the llms.txt eval 3 times on Sonnet 5.5, on this PR's preview and the live docs diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 7602f218c67755..de148332904008 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -1,18 +1,17 @@ # Runs the getsentry/sentry-for-ai-evals suites against this PR's docs. # -# Label a PR `ai-evals:` to run the tests with that pytest mark, e.g. -# `ai-evals:products`, on each target (a harness with a model) that repo's plan-evals -# action lists, or `ai-evals::` to run them on the targets whose -# `:` contains ``, and a last `:x` runs each test that many -# times, e.g. `ai-evals:llms_txt:sonnet-5.5:x3` for three runs on Sonnet 5.5 of the test -# that starts from llms.txt with no Sentry skills or MCP, only the docs. The labels are in -# .github/labels.yml, since sync-labels.yml deletes any label that isn't. Each label runs -# with two sets of docs and the same skills, from sentry-for-ai's `main`: the head reads -# docs.sentry.io from the PR's Vercel preview and the base from the production deploy of -# the PR's merge base. The plan job creates one run per label and side on -# evals.sentry.dev, and a job per test, target and side sends that test's result into it, -# each test in a clean container and all of them at once. A new push reruns every eval -# label still on the PR. +# Label a PR `ai-evals:` to run the tests with that pytest mark on each target (a +# harness with a model) that repo's plan-evals action lists, or `ai-evals::` +# to run them on the targets whose `:` contains ``, and a last +# `:x` runs each test that many times, e.g. `ai-evals:llms_txt:sonnet-5.5:x3` for +# three runs on Sonnet 5.5 of the test that starts from llms.txt with no Sentry skills or +# MCP, only the docs. The labels are in .github/labels.yml, since sync-labels.yml deletes +# any label that isn't. Each label runs with two sets of docs and the same skills, from +# sentry-for-ai's `main`: the head reads docs.sentry.io from the PR's Vercel preview and +# the base from the production deploy of the PR's merge base. The plan job creates one run +# per label and side on evals.sentry.dev, and a job per test, target and side sends that +# test's result into it, each test in a clean container and all of them at once. A new +# push reruns every eval label still on the PR. # # A production deploy redirects every page to docs.sentry.io unless the request has the # x-sentry-evals-pin header that the evals' docs proxy sends (middleware.ts). Deploys built From bc6c4680d3799ec09e35a9593b0f63e93317821e Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Mon, 26 Oct 2026 14:29:53 -0500 Subject: [PATCH 17/17] ci(evals): Say the llms.txt label's base is the merge base's docs The base reads the merge base's production deploy since #20225, not the live docs. --- .github/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/labels.yml b/.github/labels.yml index a3eaab2c3df7ca..5b60528a7270ff 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -287,7 +287,7 @@ # AI evals - .github/workflows/ai-evals.yml - name: 'ai-evals:llms_txt:sonnet-5.5:x3' color: 6F42C1 - description: Run the llms.txt eval 3 times on Sonnet 5.5, on this PR's preview and the live docs + description: Run the llms.txt eval 3 times on Sonnet 5.5, on this PR's preview and its merge base's docs # Miscellaneous - name: API Docs