Skip to content

Commit 2880308

Browse files
svelderrainruizweb-flow
svelderrainruiz
and
web-flow
authored
#867 Prove validation approval apply mode (#891)
* #867 Graduate validation approval apply mode * #867 Simplify proof fixture review count --------- Co-authored-by: svelderrainruiz <noreply@github.com>
1 parent eea81bb commit 2880308

10 files changed

Lines changed: 2014 additions & 7 deletions

‎docs/VALIDATION_APPROVAL_APPLY_PROOF.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Validation Approval Apply-Mode Proof
2+
3+
`validation` is the only environment in scope for the approval broker/apply path.
4+
`production`, release, and publish environments stay out of scope.
5+
6+
## Proof command
7+
8+
Run the historical replay against recent real `validation` deployments:
9+
10+
```bash
11+
node tools/npm/run-script.mjs priority:validation:proof -- \
12+
--repo LabVIEW-Community-CI-CD/compare-vi-cli-action \
13+
--max-deployments 8 \
14+
--min-samples 4 \
15+
--lookback-days 7
16+
```
17+
18+
The report lands at
19+
`tests/results/_agent/approvals/validation-approval-proof.json` and stages
20+
per-sample replay artifacts under `tests/results/_agent/approvals/proof/`.
21+
22+
## Graduation rule
23+
24+
Apply mode may be enabled for `validation` only when the proof report shows:
25+
26+
- `falseReadyCount = 0`
27+
- `samplesEvaluated >= minSamples`
28+
- `errorCount = 0`
29+
30+
Historical replay is intentionally conservative. The derived replay attestation
31+
never invents dispositions for unresolved/actionable Copilot comments. If a
32+
historical PR still presents stale or actionable review state, the broker must
33+
block and the sample is recorded as a conservative `false-blocked` outcome
34+
rather than a `false-ready` outcome.

‎docs/schemas/validation-approval-proof-v1.schema.json‎

Lines changed: 318 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,318 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"$id": "https://labview-community-ci-cd.github.io/compare-vi-cli-action/schemas/validation-approval-proof-v1.schema.json",
4+
"title": "validation-approval-proof@v1",
5+
"type": "object",
6+
"additionalProperties": false,
7+
"required": [
8+
"schema",
9+
"schemaVersion",
10+
"generatedAt",
11+
"status",
12+
"strict",
13+
"inputs",
14+
"proofWindow",
15+
"summary",
16+
"verdict",
17+
"samples",
18+
"skipped",
19+
"errors"
20+
],
21+
"properties": {
22+
"schema": {
23+
"const": "validation-approval-proof@v1"
24+
},
25+
"schemaVersion": {
26+
"type": "string"
27+
},
28+
"generatedAt": {
29+
"type": "string",
30+
"format": "date-time"
31+
},
32+
"status": {
33+
"type": "string",
34+
"enum": ["pass", "warn", "fail"]
35+
},
36+
"strict": {
37+
"type": "boolean"
38+
},
39+
"inputs": {
40+
"type": "object",
41+
"additionalProperties": false,
42+
"required": [
43+
"repository",
44+
"environment",
45+
"policyPath",
46+
"maxDeployments",
47+
"minSamples",
48+
"lookbackDays",
49+
"artifactsDir",
50+
"reportPath"
51+
],
52+
"properties": {
53+
"repository": {
54+
"type": "string",
55+
"pattern": "^[^/\\s]+/[^/\\s]+$"
56+
},
57+
"environment": {
58+
"type": "string",
59+
"minLength": 1
60+
},
61+
"policyPath": {
62+
"type": "string",
63+
"minLength": 1
64+
},
65+
"maxDeployments": {
66+
"type": "integer",
67+
"minimum": 1
68+
},
69+
"minSamples": {
70+
"type": "integer",
71+
"minimum": 1
72+
},
73+
"lookbackDays": {
74+
"type": "integer",
75+
"minimum": 0
76+
},
77+
"artifactsDir": {
78+
"type": "string",
79+
"minLength": 1
80+
},
81+
"reportPath": {
82+
"type": "string",
83+
"minLength": 1
84+
}
85+
}
86+
},
87+
"proofWindow": {
88+
"type": "object",
89+
"additionalProperties": false,
90+
"required": ["startedAt", "endedAt"],
91+
"properties": {
92+
"startedAt": {
93+
"$ref": "#/$defs/nullableDateTime"
94+
},
95+
"endedAt": {
96+
"$ref": "#/$defs/nullableDateTime"
97+
}
98+
}
99+
},
100+
"summary": {
101+
"type": "object",
102+
"additionalProperties": false,
103+
"required": [
104+
"deploymentsFetched",
105+
"samplesEvaluated",
106+
"skippedCount",
107+
"errorCount",
108+
"readyCount",
109+
"approvedCount",
110+
"falseReadyCount",
111+
"falseBlockedCount"
112+
],
113+
"properties": {
114+
"deploymentsFetched": { "type": "integer", "minimum": 0 },
115+
"samplesEvaluated": { "type": "integer", "minimum": 0 },
116+
"skippedCount": { "type": "integer", "minimum": 0 },
117+
"errorCount": { "type": "integer", "minimum": 0 },
118+
"readyCount": { "type": "integer", "minimum": 0 },
119+
"approvedCount": { "type": "integer", "minimum": 0 },
120+
"falseReadyCount": { "type": "integer", "minimum": 0 },
121+
"falseBlockedCount": { "type": "integer", "minimum": 0 }
122+
}
123+
},
124+
"verdict": {
125+
"type": "object",
126+
"additionalProperties": false,
127+
"required": ["policyFlipRecommended", "reasons", "summary"],
128+
"properties": {
129+
"policyFlipRecommended": {
130+
"type": "boolean"
131+
},
132+
"reasons": {
133+
"type": "array",
134+
"items": {
135+
"type": "string",
136+
"minLength": 1
137+
}
138+
},
139+
"summary": {
140+
"type": "string",
141+
"minLength": 1
142+
}
143+
}
144+
},
145+
"samples": {
146+
"type": "array",
147+
"items": {
148+
"$ref": "#/$defs/sample"
149+
}
150+
},
151+
"skipped": {
152+
"type": "array",
153+
"items": {
154+
"$ref": "#/$defs/skippedSample"
155+
}
156+
},
157+
"errors": {
158+
"type": "array",
159+
"items": {
160+
"$ref": "#/$defs/errorSample"
161+
}
162+
}
163+
},
164+
"$defs": {
165+
"nullableString": {
166+
"type": ["string", "null"]
167+
},
168+
"nullableDateTime": {
169+
"type": ["string", "null"],
170+
"format": "date-time"
171+
},
172+
"sample": {
173+
"type": "object",
174+
"additionalProperties": false,
175+
"required": [
176+
"deploymentId",
177+
"runId",
178+
"runUrl",
179+
"pullRequestNumber",
180+
"pullRequestUrl",
181+
"ref",
182+
"sha",
183+
"createdAt",
184+
"latestStatusAt",
185+
"finalState",
186+
"waitedForApproval",
187+
"actualDecision",
188+
"brokerState",
189+
"brokerReady",
190+
"comparison",
191+
"reasons",
192+
"notes",
193+
"reviewSignal",
194+
"deploymentDeterminism",
195+
"artifacts"
196+
],
197+
"properties": {
198+
"deploymentId": { "type": "integer", "minimum": 1 },
199+
"runId": { "type": ["integer", "null"], "minimum": 1 },
200+
"runUrl": { "$ref": "#/$defs/nullableString" },
201+
"pullRequestNumber": { "type": "integer", "minimum": 1 },
202+
"pullRequestUrl": { "$ref": "#/$defs/nullableString" },
203+
"ref": { "$ref": "#/$defs/nullableString" },
204+
"sha": {
205+
"type": ["string", "null"],
206+
"pattern": "^[0-9a-f]{40}$"
207+
},
208+
"createdAt": { "$ref": "#/$defs/nullableDateTime" },
209+
"latestStatusAt": { "$ref": "#/$defs/nullableDateTime" },
210+
"finalState": { "$ref": "#/$defs/nullableString" },
211+
"waitedForApproval": { "type": "boolean" },
212+
"actualDecision": {
213+
"type": "string",
214+
"enum": ["approved", "not-approved"]
215+
},
216+
"brokerState": {
217+
"type": "string",
218+
"enum": ["ready", "blocked", "denied", "error"]
219+
},
220+
"brokerReady": { "type": "boolean" },
221+
"comparison": {
222+
"type": "string",
223+
"enum": ["match-ready", "match-not-ready", "false-ready", "false-blocked"]
224+
},
225+
"reasons": {
226+
"type": "array",
227+
"items": { "type": "string", "minLength": 1 }
228+
},
229+
"notes": {
230+
"type": "array",
231+
"items": { "type": "string", "minLength": 1 }
232+
},
233+
"reviewSignal": {
234+
"type": "object",
235+
"additionalProperties": false,
236+
"required": [
237+
"status",
238+
"reviewState",
239+
"hasCurrentHeadReview",
240+
"actionableCommentCount",
241+
"unresolvedThreadCount",
242+
"staleReviewCount"
243+
],
244+
"properties": {
245+
"status": { "$ref": "#/$defs/nullableString" },
246+
"reviewState": { "$ref": "#/$defs/nullableString" },
247+
"hasCurrentHeadReview": { "type": "boolean" },
248+
"actionableCommentCount": { "type": "integer", "minimum": 0 },
249+
"unresolvedThreadCount": { "type": "integer", "minimum": 0 },
250+
"staleReviewCount": { "type": "integer", "minimum": 0 }
251+
}
252+
},
253+
"deploymentDeterminism": {
254+
"type": "object",
255+
"additionalProperties": false,
256+
"required": ["source", "result", "issueCount", "artifactPath"],
257+
"properties": {
258+
"source": {
259+
"type": "string",
260+
"enum": ["artifact", "fallback"]
261+
},
262+
"result": { "$ref": "#/$defs/nullableString" },
263+
"issueCount": { "type": "integer", "minimum": 0 },
264+
"artifactPath": { "type": "string", "minLength": 1 }
265+
}
266+
},
267+
"artifacts": {
268+
"type": "object",
269+
"additionalProperties": false,
270+
"required": [
271+
"sampleDir",
272+
"signalPath",
273+
"pullPath",
274+
"attestationPath",
275+
"decisionPath",
276+
"eventsPath",
277+
"deploymentDeterminismPath"
278+
],
279+
"properties": {
280+
"sampleDir": { "type": "string", "minLength": 1 },
281+
"signalPath": { "type": "string", "minLength": 1 },
282+
"pullPath": { "type": "string", "minLength": 1 },
283+
"attestationPath": { "type": "string", "minLength": 1 },
284+
"decisionPath": { "type": "string", "minLength": 1 },
285+
"eventsPath": { "type": "string", "minLength": 1 },
286+
"deploymentDeterminismPath": { "type": "string", "minLength": 1 }
287+
}
288+
}
289+
}
290+
},
291+
"skippedSample": {
292+
"type": "object",
293+
"additionalProperties": false,
294+
"required": ["deploymentId", "reason", "ref", "sha"],
295+
"properties": {
296+
"deploymentId": { "type": ["integer", "null"], "minimum": 1 },
297+
"runId": { "type": ["integer", "null"], "minimum": 1 },
298+
"reason": { "type": "string", "minLength": 1 },
299+
"ref": { "$ref": "#/$defs/nullableString" },
300+
"sha": {
301+
"type": ["string", "null"],
302+
"pattern": "^[0-9a-f]{40}$"
303+
}
304+
}
305+
},
306+
"errorSample": {
307+
"type": "object",
308+
"additionalProperties": false,
309+
"required": ["deploymentId", "runId", "pullRequestNumber", "message"],
310+
"properties": {
311+
"deploymentId": { "type": ["integer", "null"], "minimum": 1 },
312+
"runId": { "type": ["integer", "null"], "minimum": 1 },
313+
"pullRequestNumber": { "type": ["integer", "null"], "minimum": 1 },
314+
"message": { "type": "string", "minLength": 1 }
315+
}
316+
}
317+
}
318+
}

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
"priority:validation:attestation": "tsc -p tsconfig.json && node dist/tools/priority/validation-agent-attestation.js",
6363
"priority:validation:broker": "node tools/priority/validation-approval-broker.mjs",
6464
"priority:validation:helper": "node tools/priority/validation-approval-helper.mjs",
65+
"priority:validation:proof": "node tools/npm/run-script.mjs build && node tools/priority/validation-approval-proof.mjs",
6566
"priority:merge-sync": "node tools/priority/merge-sync-pr.mjs",
6667
"priority:event:ingest": "node tools/priority/event-ingest.mjs",
6768
"priority:decision:ledger": "node tools/priority/decision-ledger.mjs",
@@ -122,6 +123,7 @@
122123
"schema:copilot-review-signal:validate": "node tools/npm/run-script.mjs schema:validate -- --schema docs/schemas/copilot-review-signal-v1.schema.json --data tests/results/_agent/reviews/copilot-review-signal.json --optional",
123124
"schema:validation-agent-attestation:validate": "node tools/npm/run-script.mjs schema:validate -- --schema docs/schemas/validation-agent-attestation-v1.schema.json --data tests/results/_agent/reviews/validation-agent-attestation.json --optional",
124125
"schema:validation-approval-decision:validate": "node tools/npm/run-script.mjs schema:validate -- --schema docs/schemas/validation-approval-decision-v1.schema.json --data tests/results/_agent/approvals/validation-approval-decision.json --optional",
126+
"schema:validation-approval-proof:validate": "node tools/npm/run-script.mjs schema:validate -- --schema docs/schemas/validation-approval-proof-v1.schema.json --data tests/results/_agent/approvals/validation-approval-proof.json --optional",
125127
"schema:watcher:validate": "node tools/npm/run-script.mjs schema:validate -- --schema docs/schemas/watcher-telemetry.v1.schema.json --data tests/results/_agent/handoff/watcher-telemetry.json --optional",
126128
"schemas:generate": "tsc -p tsconfig.cli.json && node dist/tools/schemas/generate-schemas.js",
127129
"semver:check": "node tools/priority/validate-semver.mjs",

‎tools/policy/validation-approval-policy.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"schema": "validation-approval-policy/v1",
33
"schemaVersion": "1.0.0",
44
"environment": "validation",
5-
"shadowMode": true,
5+
"shadowMode": false,
66
"allowedBaseRefs": [
77
"develop"
88
],

‎tools/priority/__tests__/validation-approval-broker.test.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ test('validation approval broker returns ready for trusted clean inputs', async
188188

189189
assert.equal(result.exitCode, 0);
190190
assert.equal(result.report?.decision.state, 'ready');
191-
assert.deepEqual(result.report?.decision.reasons, ['approval-ready-shadow-mode']);
191+
assert.deepEqual(result.report?.decision.reasons, ['approval-ready']);
192192

193193
const lines = (await readFile(eventsPath, 'utf8')).trim().split(/\r?\n/);
194194
assert.ok(lines.length >= 2);

0 commit comments

Comments
 (0)