azure-sdk/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml at 763204e9934a499efa2190bd3014b95171a7faa6 · Azure/azure-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
name: azsdk-common-pipeline-fixer-trigger-eval
description: Trigger and anti-trigger tests for the azsdk-common-pipeline-fixer skill
type: capability

environment: azsdk-mcp-mock

tags:
  area: azsdk-common-pipeline-fixer
  type: ci-gate

defaults:
  runs: 1
  # Stimuli with a resolvable build reference (URL or build ID) make the agent
  # run the full analyze→fix→verify workflow (~90-180s under gpt-5.4),
  # so the per-stimulus cap matches the Vally scenario evals (5m). Non-build-ref
  # routing prompts still complete in ~10-20s.
  timeout: "5m"
  model: gpt-5.4
  executor: copilot-sdk

scoring:
  threshold: 0.8

stimuli:
  # Should trigger
  - name: trigger-fix-pipeline-failure
    prompt: "fix the pipeline failure for build 6447834"
    environment: &targetSkill
      skills:
        - ..
    graders:
      - type: skill-invocation
        config:
          required: ["azsdk-common-pipeline-fixer"]
  - name: trigger-auto-fix-failing-tests
    prompt: "My Azure SDK PR has failing tests in CI: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=6447834. Can you fix them?"
    environment: *targetSkill
    graders:
      - type: skill-invocation
        config:
          required: ["azsdk-common-pipeline-fixer"]
  - name: trigger-resolve-pipeline-failure
    prompt: "resolve my Azure SDK pipeline failure"
    environment: *targetSkill
    graders:
      - type: skill-invocation
        config:
          required: ["azsdk-common-pipeline-fixer"]
  - name: trigger-fix-build-error
    prompt: "fix the build error in my Azure SDK CI"
    environment: *targetSkill
    graders:
      - type: skill-invocation
        config:
          required: ["azsdk-common-pipeline-fixer"]
  - name: trigger-fix-mypy-type-error
    prompt: "My Python SDK CI is failing with a mypy type error: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=6447834. Can you fix it?"
    environment: *targetSkill
    graders:
      - type: skill-invocation
        config:
          required: ["azsdk-common-pipeline-fixer"]

  # Should NOT trigger
  # Analysis-only prompts belong to azsdk-common-pipeline-analysis; this
  # skill applies + verifies fixes and must stay out of pure-diagnosis requests.
  - name: anti-trigger-analyze-only
    prompt: "what went wrong in my pipeline build 12345? just tell me, don't fix anything"
    environment: &analysisBoundary
      skills:
        - ..
        - ../../azsdk-common-pipeline-analysis
    graders:
      - type: skill-invocation
        config:
          required: ["azsdk-common-pipeline-analysis"]
          disallowed: ["azsdk-common-pipeline-fixer"]
  - name: anti-trigger-why-is-ci-red
    prompt: "Azure SDK pipeline build 6447834 is red. Analyze the root cause; do not change any code."
    environment: *analysisBoundary
    graders:
      - type: skill-invocation
        config:
          required: ["azsdk-common-pipeline-analysis"]
          disallowed: ["azsdk-common-pipeline-fixer"]
  - name: anti-trigger-write-a-typespec-definition
    prompt: "Add an optional color property to the WidgetSuite model in main.tsp."
    environment:
      skills:
        - ..
        - ../../azure-typespec-author
    graders:
      - type: skill-invocation
        config:
          required: ["azure-typespec-author"]
          disallowed: ["azsdk-common-pipeline-fixer"]
  - name: anti-trigger-release-sdk-package
    prompt: "Check release readiness and trigger the SDK release pipeline for release plan 12345."
    environment:
      skills:
        - ..
        - ../../azsdk-common-sdk-release
    graders:
      - type: skill-invocation
        config:
          required: ["azsdk-common-sdk-release"]
          disallowed: ["azsdk-common-pipeline-fixer"]
  - name: anti-trigger-create-a-new-skill
    prompt: "create a new skill"
    environment:
      skills:
        - ..
        - ../../skill-authoring
    graders:
      - type: skill-invocation
        config:
          required: ["skill-authoring"]
          disallowed: ["azsdk-common-pipeline-fixer"]