From 763204e9934a499efa2190bd3014b95171a7faa6 Mon Sep 17 00:00:00 2001 From: helen229 Date: Thu, 3 Sep 2026 22:50:45 -0700 Subject: [PATCH] Fix skill eval environment routing --- .../evals/trigger.eval.yaml | 31 +++++++++++++- .../evals/trigger.eval.yaml | 35 +++++++++++++++- .../evals/trigger.eval.yaml | 35 ++++++++++++++-- .../evals/trigger.eval.yaml | 26 +++++++++++- .../evals/trigger.eval.yaml | 40 ++++++++++++++++--- 5 files changed, 154 insertions(+), 13 deletions(-) diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger.eval.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger.eval.yaml index 7943af10c966..4baee7fd21fd 100644 --- a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger.eval.yaml @@ -20,48 +20,58 @@ scoring: stimuli: - name: trigger-resolve-apiview-feedback-on-my-pr prompt: "resolve APIView feedback on my PR" + environment: &targetSkill + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-fix-apiview-comments prompt: "fix APIView comments" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-address-api-review-feedback prompt: "address API review feedback" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-apiview-shows-issues-with-my-sdk prompt: "APIView shows issues with my SDK" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-help-me-with-apiview-suggestions prompt: "help me with APIView suggestions" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-fix-comments-apiview-url-1 prompt: "Help me fix these comments: https://spa.apiview.dev/review/0qw98r9qwur9q823r?activeApiRevisionId=afiq948tquajf8jq9f" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-address-feedback-apiview-url-2 prompt: "Address this feedback: https://spa.apiview.dev/review/akldfja9qjta498tjaejg0?activeApiRevisionId=odiufa9sefighj98439" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-resolve-comments-staging-url prompt: "Resolve these comments: https://apiviewstagingtest.com/review/akldfja9qjta498tjaejg0?activeApiRevisionId=odiufa9sefighj98439" + environment: *targetSkill graders: - type: skill-invocation config: @@ -69,30 +79,47 @@ stimuli: - name: anti-trigger-deploy-my-azure-function prompt: "deploy my Azure Function" + environment: *targetSkill graders: - type: skill-invocation config: disallowed: ["azsdk-common-apiview-feedback-resolution"] - name: anti-trigger-fix-my-pipeline-build-error prompt: "fix my pipeline build error" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-fixer graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-fixer"] disallowed: ["azsdk-common-apiview-feedback-resolution"] - name: anti-trigger-create-a-new-typespec-definition - prompt: "create a new TypeSpec definition" + prompt: "Add an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. + - ../../azure-typespec-author graders: - type: skill-invocation config: + required: ["azure-typespec-author"] disallowed: ["azsdk-common-apiview-feedback-resolution"] - name: anti-trigger-how-do-i-release-an-sdk-package - prompt: "how do I release an SDK package" + prompt: "Check release readiness and trigger the SDK release pipeline for release plan 12345." + environment: + skills: + - .. + - ../../azsdk-common-sdk-release graders: - type: skill-invocation config: + required: ["azsdk-common-sdk-release"] disallowed: ["azsdk-common-apiview-feedback-resolution"] - name: anti-trigger-set-up-azure-storage-account prompt: "set up Azure Storage account" + environment: *targetSkill graders: - type: skill-invocation config: diff --git a/.github/skills/azsdk-common-pipeline-analysis/evals/trigger.eval.yaml b/.github/skills/azsdk-common-pipeline-analysis/evals/trigger.eval.yaml index 5fedb0dedbfc..ec7e3d5c3dc7 100644 --- a/.github/skills/azsdk-common-pipeline-analysis/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-pipeline-analysis/evals/trigger.eval.yaml @@ -21,30 +21,37 @@ stimuli: # Should trigger - name: trigger-my-ci-pipeline-is-failing prompt: "my Azure SDK CI pipeline build 6447834 is failing, can you help troubleshoot what went wrong?" + environment: &targetSkill + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-analysis"] - name: trigger-debug-pipeline-build-error prompt: "debug the build error in my SDK generation pipeline, build 6447834" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-analysis"] - name: trigger-pipeline-analysis-help prompt: "I'm stuck on a failing Azure SDK pipeline — can you help me troubleshoot it?" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-analysis"] - name: trigger-analyze-failing-sdk-pipeline prompt: "analyze my failing Azure SDK generation pipeline https://dev.azure.com/azure-sdk/internal/_build/results?buildId=6447834 and tell me what went wrong" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-analysis"] - name: trigger-ci-build-broke-after-my-change prompt: "the SDK CI build broke after my change, help me troubleshoot the pipeline" + environment: *targetSkill graders: - type: skill-invocation config: @@ -55,37 +62,63 @@ stimuli: # this analysis-only skill (see each skill's DO NOT USE FOR boundary). - name: anti-trigger-auto-fix-and-commit prompt: "auto-fix the failing tests in my pipeline and commit the changes to my PR" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-fixer graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-fixer"] disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-write-a-typespec-definition prompt: "write a TypeSpec definition" + environment: *targetSkill graders: - type: skill-invocation config: disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-create-a-new-skill prompt: "create a new skill" + environment: + skills: + - .. + - ../../skill-authoring graders: - type: skill-invocation config: + required: ["skill-authoring"] disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-optimize-markdown-tokens prompt: "optimize markdown tokens" + environment: + skills: + - .. + - ../../markdown-token-optimizer graders: - type: skill-invocation config: + required: ["markdown-token-optimizer"] disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-how-do-i-release-an-sdk-package - prompt: "how do I release an SDK package" + prompt: "Check release readiness and trigger the SDK release pipeline for release plan 12345." + environment: + skills: + - .. + - ../../azsdk-common-sdk-release graders: - type: skill-invocation config: + required: ["azsdk-common-sdk-release"] disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-resolve-apiview-feedback prompt: "resolve APIView feedback" + environment: + skills: + - .. + - ../../azsdk-common-apiview-feedback-resolution graders: - type: skill-invocation config: + required: ["azsdk-common-apiview-feedback-resolution"] disallowed: ["azsdk-common-pipeline-analysis"] diff --git a/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml b/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml index 1709a50e2624..1c769875c35f 100644 --- a/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml @@ -25,30 +25,37 @@ stimuli: # Should trigger - name: trigger-fix-pipeline-failure prompt: "fix the pipeline failure for build 6447834" + environment: &targetSkill + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-fixer"] - name: trigger-auto-fix-failing-tests prompt: "My Azure SDK PR has failing tests in CI: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=6447834. Can you fix them?" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-fixer"] - name: trigger-resolve-pipeline-failure prompt: "resolve my Azure SDK pipeline failure" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-fixer"] - name: trigger-fix-build-error prompt: "fix the build error in my Azure SDK CI" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-fixer"] - name: trigger-fix-mypy-type-error prompt: "My Python SDK CI is failing with a mypy type error: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=6447834. Can you fix it?" + environment: *targetSkill graders: - type: skill-invocation config: @@ -59,31 +66,53 @@ stimuli: # skill applies + verifies fixes and must stay out of pure-diagnosis requests. - name: anti-trigger-analyze-only prompt: "what went wrong in my pipeline build 12345? just tell me, don't fix anything" + environment: &analysisBoundary + skills: + - .. + - ../../azsdk-common-pipeline-analysis graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-analysis"] disallowed: ["azsdk-common-pipeline-fixer"] - name: anti-trigger-why-is-ci-red - prompt: "why is my CI red? just diagnose the root cause, do not change any code" + prompt: "Azure SDK pipeline build 6447834 is red. Analyze the root cause; do not change any code." + environment: *analysisBoundary graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-analysis"] disallowed: ["azsdk-common-pipeline-fixer"] - name: anti-trigger-write-a-typespec-definition - prompt: "write a TypeSpec definition" + prompt: "Add an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. + - ../../azure-typespec-author graders: - type: skill-invocation config: + required: ["azure-typespec-author"] disallowed: ["azsdk-common-pipeline-fixer"] - name: anti-trigger-release-sdk-package - prompt: "how do I release an SDK package" + prompt: "Check release readiness and trigger the SDK release pipeline for release plan 12345." + environment: + skills: + - .. + - ../../azsdk-common-sdk-release graders: - type: skill-invocation config: + required: ["azsdk-common-sdk-release"] disallowed: ["azsdk-common-pipeline-fixer"] - name: anti-trigger-create-a-new-skill prompt: "create a new skill" + environment: + skills: + - .. + - ../../skill-authoring graders: - type: skill-invocation config: + required: ["skill-authoring"] disallowed: ["azsdk-common-pipeline-fixer"] diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/trigger.eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/trigger.eval.yaml index 06f7afab8b14..02141eaea26f 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/trigger.eval.yaml @@ -20,30 +20,37 @@ scoring: stimuli: - name: trigger-prepare-a-release-plan prompt: "prepare a release plan" + environment: &targetSkill + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-prepare-release-plan"] - name: trigger-plan-the-next-sdk-release prompt: "plan the next SDK release" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-prepare-release-plan"] - name: trigger-create-release-plan-for-my-package prompt: "create release plan for my package" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-prepare-release-plan"] - name: trigger-what-goes-in-a-release-plan prompt: "create a release plan for my new Azure SDK service" + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-prepare-release-plan"] - name: trigger-help-me-prepare-for-sdk-release prompt: "help me prepare a release plan for my new Azure SDK package" + environment: *targetSkill graders: - type: skill-invocation config: @@ -51,31 +58,48 @@ stimuli: - name: anti-trigger-deploy-my-azure-function prompt: "deploy my Azure Function" + environment: *targetSkill graders: - type: skill-invocation config: disallowed: ["azsdk-common-prepare-release-plan"] - name: anti-trigger-fix-my-pipeline-build-error prompt: "fix my pipeline build error" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-fixer graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-fixer"] disallowed: ["azsdk-common-prepare-release-plan"] - name: anti-trigger-create-a-new-typespec-definition - prompt: "create a new TypeSpec definition" + prompt: "Add an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. + - ../../azure-typespec-author graders: - type: skill-invocation config: + required: ["azure-typespec-author"] disallowed: ["azsdk-common-prepare-release-plan"] - name: anti-trigger-configure-azure-logging prompt: "configure Azure logging" + environment: *targetSkill graders: - type: skill-invocation config: disallowed: ["azsdk-common-prepare-release-plan"] - name: anti-trigger-resolve-apiview-feedback prompt: "resolve APIView feedback" + environment: + skills: + - .. + - ../../azsdk-common-apiview-feedback-resolution graders: - type: skill-invocation config: + required: ["azsdk-common-apiview-feedback-resolution"] disallowed: ["azsdk-common-prepare-release-plan"] diff --git a/.github/skills/azsdk-common-sdk-release/evals/trigger.eval.yaml b/.github/skills/azsdk-common-sdk-release/evals/trigger.eval.yaml index 92c765f55703..a6fb4544cc66 100644 --- a/.github/skills/azsdk-common-sdk-release/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-sdk-release/evals/trigger.eval.yaml @@ -19,31 +19,38 @@ scoring: stimuli: - name: trigger-release-my-sdk-package - prompt: "release my SDK package" + prompt: "Check release readiness and release my SDK package for release plan 12345." + environment: &targetSkill + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-sdk-release"] - name: trigger-how-to-publish-a-new-sdk-version - prompt: "how to publish a new SDK version" + prompt: "Check whether release plan 12345 is ready, then publish its SDK package." + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-sdk-release"] - name: trigger-walk-me-through-sdk-release-process - prompt: "walk me through SDK release process" + prompt: "Walk me through and trigger the SDK release process for release plan 12345." + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-sdk-release"] - name: trigger-release-a-new-version-of-azure-sdk - prompt: "release a new version of azure-sdk" + prompt: "Trigger the Azure SDK release pipeline for release plan 12345." + environment: *targetSkill graders: - type: skill-invocation config: required: ["azsdk-common-sdk-release"] - name: trigger-sdk-release-checklist - prompt: "SDK release checklist" + prompt: "Run the SDK release readiness checklist for release plan 12345." + environment: *targetSkill graders: - type: skill-invocation config: @@ -51,31 +58,52 @@ stimuli: - name: anti-trigger-write-unit-tests prompt: "write unit tests" + environment: *targetSkill graders: - type: skill-invocation config: disallowed: ["azsdk-common-sdk-release"] - name: anti-trigger-fix-my-pipeline-build-error prompt: "fix my pipeline build error" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-fixer graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-fixer"] disallowed: ["azsdk-common-sdk-release"] - name: anti-trigger-create-a-new-typespec-definition - prompt: "create a new TypeSpec definition" + prompt: "Add an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. + - ../../azure-typespec-author graders: - type: skill-invocation config: + required: ["azure-typespec-author"] disallowed: ["azsdk-common-sdk-release"] - name: anti-trigger-optimize-markdown-tokens prompt: "optimize markdown tokens" + environment: + skills: + - .. + - ../../markdown-token-optimizer graders: - type: skill-invocation config: + required: ["markdown-token-optimizer"] disallowed: ["azsdk-common-sdk-release"] - name: anti-trigger-resolve-apiview-feedback prompt: "resolve APIView feedback" + environment: + skills: + - .. + - ../../azsdk-common-apiview-feedback-resolution graders: - type: skill-invocation config: + required: ["azsdk-common-apiview-feedback-resolution"] disallowed: ["azsdk-common-sdk-release"]