Skip to content

Commit 5fbcb78

Browse files
comet-qa-botclaude
andcommitted
[QA] review: name the mapper branch in row failures; scope the edit-rule note
- json-looking-judge-prompts.spec.ts: the JsonLookingPrompt.why field was populated and never read, so a missing row failed with "row for <name>" and the reader had to go back to the table to learn which mapper branch that shape reaches. Carry name+why through the UI step and put it in the message. - taxonomy.yaml: the edit-rule note read as though the edit dialog was driven for both prompt shapes. Only the plain-string half is; the content_array half is API-only, because the dialog renders no image part. Scoped the note to say so, and recorded the open question — the content_array test asserts content-shape preservation, which no capability in this area names, so it rides on edit-rule until a human decides whether to add a key or drop the tag. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 38cc35d commit 5fbcb78

2 files changed

Lines changed: 29 additions & 10 deletions

File tree

tests_end_to_end/coverage/taxonomy.yaml

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -661,11 +661,23 @@ areas:
661661
rule-filters: { covered: false }
662662
sampling-rate: { covered: true, tier: t2-cuj, note: "50% rule vs 100% control over one 30-trace batch, binomial band 15-85%; plus a 0%-rate rule at trigger_scope=both, which must skip every SDK trace and still score experiment/playground/optimization ones" }
663663
clone-rule: { covered: false }
664-
# Scoped: the edit dialog is opened, its hydrated prompt asserted byte-exact,
665-
# and submitted unchanged so the save round trip is real. Editing a FIELD
666-
# through the dialog (renaming, changing the model, rewriting the prompt) is
667-
# still uncovered.
668-
edit-rule: { covered: true, tier: t2-cuj, note: "dialog hydration + no-op re-save preserve a judge prompt byte-for-byte, in both the plain-string and content_array shapes; editing a field through the dialog is not covered" }
664+
# Scoped, and the two halves are not equally scoped:
665+
# - plain-string prompt: the edit dialog IS opened, its hydrated prompt
666+
# asserted byte-exact, and submitted unchanged, so the UI save round
667+
# trip is real.
668+
# - content_array prompt: API-level only. The dialog renders no image
669+
# part, so the url/detail that a regression would drop are not
670+
# observable in the UI; that half is a read-then-write over REST and
671+
# no dialog gesture is made.
672+
# Editing a FIELD through the dialog (renaming, changing the model,
673+
# rewriting the prompt) is still uncovered either way.
674+
#
675+
# OPEN, needs a human call: the content_array test asserts judge-message
676+
# content-SHAPE preservation, which no capability in this area names. It
677+
# rides on edit-rule for want of a better key. Either add a capability for
678+
# it or drop that tag and leave the behaviour honestly uncovered — do not
679+
# let it sit here as a silent widening of edit-rule.
680+
edit-rule: { covered: true, tier: t2-cuj, note: "plain-string prompt: edit-dialog hydration + no-op re-save, byte-for-byte; content_array prompt: API-only read-then-write re-save, no dialog gesture; editing a field through the dialog is not covered" }
669681
enable-disable-rule: { covered: true, tier: t2-cuj, note: "edit-dialog switch; control rule proves scoring stopped, then resumed" }
670682
delete-rule: { covered: true, tier: t2-cuj, note: "row kebab delete; control rule proves scoring stopped" }
671683
# Deliberately still false. online-evaluation-python-metric-errors.spec.ts

tests_end_to_end/e2e/tests/online-evaluation/online-evaluation-json-looking-judge-prompts.spec.ts

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,17 @@ test.describe('Online Evaluation — judge prompts that look like JSON', { tag:
7373
automationRulesCleanup,
7474
}) => {
7575
const controlName = `${testNamespace}-control`;
76-
const seededNames = [
77-
controlName,
78-
...JSON_LOOKING_PROMPTS.map((p) => `${testNamespace}-${p.label}`),
76+
// Carried as name+why rather than name alone so a missing row names the
77+
// mapper branch it came from: "row for …-empty-object missing" is a bug
78+
// report, "row missing" is a starting point for one.
79+
const seededRules: Array<{ name: string; why: string }> = [
80+
{ name: controlName, why: 'ordinary prose — the collateral-damage control' },
81+
...JSON_LOOKING_PROMPTS.map((p) => ({
82+
name: `${testNamespace}-${p.label}`,
83+
why: p.why,
84+
})),
7985
];
86+
const seededNames = seededRules.map((r) => r.name);
8087

8188
await test.step(
8289
`Seed a prose control rule plus ${JSON_LOOKING_PROMPTS.length} bracket-opening ones`,
@@ -152,8 +159,8 @@ test.describe('Online Evaluation — judge prompts that look like JSON', { tag:
152159
await onlineEval.goto(project.id);
153160
await onlineEval.waitForReady();
154161

155-
for (const name of seededNames) {
156-
await expect(onlineEval.ruleRow(name), `row for ${name}`).toHaveCount(1);
162+
for (const { name, why } of seededRules) {
163+
await expect(onlineEval.ruleRow(name), `row for ${name}${why}`).toHaveCount(1);
157164
}
158165
await expect(
159166
onlineEval.ruleRows,

0 commit comments

Comments
 (0)