-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathreport.json
More file actions
158 lines (158 loc) · 11.6 KB
/
Copy pathreport.json
File metadata and controls
158 lines (158 loc) · 11.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
{
"schema_version": 1,
"protocol_id": "independent-product-review-v1",
"status": "FAIL",
"status_reason": null,
"host": {
"runner": "codex",
"model": "gpt-5.6-sol",
"provider_family": "openai"
},
"runner_identity": {
"mode": "live",
"path": "/opt/homebrew/lib/node_modules/@openai/codex/bin/codex.js",
"sha256": "134063e133f0b4244fa3b251acf973d4fe4b4aeeacbdc135211bf480f59f1477",
"version": "codex-cli 0.144.1"
},
"model_tool_surface": "none",
"source_read_isolation": "prompt-complete-zero-tools",
"credential_environment": "parent-auth-staged-model-tools-disabled",
"execution_mode": "live",
"independent_evidence_eligible": true,
"runner_exit_code": 0,
"elapsed_ms": 192921,
"packet_path": "/private/tmp/e2e-independent-review-20260731-r6/codex/packet.json",
"packet_manifest_path": "/private/tmp/e2e-independent-review-20260731-r6/codex/packet-manifest.json",
"raw_output_path": "/private/tmp/e2e-independent-review-20260731-r6/codex/raw-codex-gpt-5.6-sol.json",
"raw_output_sha256": "38fbf0255e1c764e8f0f99b99c634532bf9d013fc15c30de49dbe503a4cf798c",
"raw_output_original_sha256": "38fbf0255e1c764e8f0f99b99c634532bf9d013fc15c30de49dbe503a4cf798c",
"raw_output_exact": true,
"integrity_before": {
"protocol_sha256": "ff5b33d26103c2a21cec91f91a086ad6420188e99f005cef90f77e3932cfd340",
"packet_sha256": "5b349530c3153c11c7abe0d30dc2073b16091bc82c7d071a6518b4bc4ef53a72",
"packet_manifest_sha256": "968b0159bf001976bf28344d604af7c43136f86df75d430e2734f791010d4701",
"independent_runner_sha256": "01d4f5907e56af599d38ec7b0c873d2940116325d6562cd441efa04dc83fcfca",
"shared_zero_tool_runner_sha256": "f2c66cd60585fcb91291559a2007213e4f4e8cea3f4979b0eb34c82a1bf3d5c0",
"selected_sources_sha256": "dea0e4b71b33afb25fe77f40c7a591f83c453ff436c94b7ddcb0efad7c0fcdba",
"selected_sources": {
"README.md": "1f17edb54748c62426b452d598f6d3779e151a8a132b25b0cf9900899ff625d2",
"SECURITY.md": "d2f29245e3b67aa6e570b269c6121424ebd1a71b1ebb01e69db4415df47038ec",
".claude-plugin/plugin.json": "63a37edb76db4b652b15806f866a2cd335a400acf8455cbc91b3e47430098525",
".claude-plugin/marketplace.json": "89006ac66d5d4eb779dcb07d0f82a18cd9b62c8636bd1b914d99122c0c00831a",
".codex-plugin/plugin.json": "a54a828f5b74f8665ff2c7384aac02accfe50643db6e0a9b84031e8dea43fa43",
"skills/playwright-test-generator/SKILL.md": "f7a29ac29549c4d156caec531111251f3410a0a6a9af06a33c7e2b6622dfd4b0",
"skills/e2e-reviewer/SKILL.md": "e4e1d489c3cc6c54c3a74b825e4e27a9c7ba371c8d9add79ade0f0ec10dbdc2c",
"skills/playwright-debugger/SKILL.md": "2f5bff15ce3219dfb3ce598798b274e3dbe14ab7351009a40ad13ca7677a5d72",
"skills/cypress-debugger/SKILL.md": "f32f38cd2f8b568b35827e4fad99e152067feba1b31fb1403ec16bb78a69cf70",
"skills/e2e-reviewer/references/pattern-reference.md": "4dc1763090cf6738c5166a336e4f4123915efcb5f3f521397e33baa621b83d7e",
"skills/e2e-reviewer/references/verification-rules.md": "09f5998f1262d33affcfb44ce417bd52c9a53909336d505fc085526581361559",
"skills/e2e-reviewer/scripts/scan.sh": "e6e266f291333d5cf6ddc259767db1e3a644963c4657a61762b90f225a60391b",
"skills/playwright-test-generator/scripts/preflight_target.py": "91b277a46fb885b3f6c2b90d31ac72a61fc10a6bd150abbc03eecbe358e9e822",
"skills/playwright-debugger/scripts/read-playwright-artifact.py": "bf16a2c15a5bd4b9710477fd91b4489f02f3aff55d6f98e25568b88d1b7128f6",
"skills/playwright-debugger/scripts/publish-json-report.py": "048e5759d1956e70e4c4ca16bb8543f9edfa0c3861d13ef11e63c24d7fe96ed6",
"skills/playwright-debugger/scripts/download-playwright-report.py": "2570d38939f6a63d0c68d4985861ac4c4710879e13ecb1c19a50a391901d2a61",
"skills/cypress-debugger/scripts/read-cypress-artifact.py": "baca00a68a81a092b7d19a4d0e0535e6aa273c7fd55caa06c721ecc7c638d0ee",
"skills/cypress-debugger/scripts/extract-junit-failures.py": "892a8cb04a84eed8f2c18821623f95e7bd7f1d9e6075cdac05e9f23320556124",
"skills/cypress-debugger/scripts/download-cypress-reports.py": "7b4fc86eb35c3973dc77bb5f71f654faffcfcdfbf1216b7ca749e001258fe091",
"skills/cypress-debugger/scripts/redact_artifact.py": "2e2c85d6219114817f9a0983c3eaf9bde971c589b043fb1997f45d72e08bad67",
"skills/playwright-test-generator/best-practices.md": "b8472249b1f2406e35775eadfe0e973114436844b1117ec804fb5a09817850db",
"skills/playwright-test-generator/code-rules.md": "f1e2567f2938166c6bee7b374ec49f4398c89a32a9143f3d8b8660f0879d3bfe",
"skills/playwright-test-generator/verification-rules.md": "8564baf915821979263cef7b6e588917f33274d9d8bf7826842f192186378216",
"skills/e2e-reviewer/references/grep-patterns.md": "f25037e73e7ae2067f01725cc9c063657c08fc51dffb611c641866ad10317136",
"scripts/ci/ci-local.sh": "85ae26710ecced7f6d46a675ad135cc03ac9ebe408b9929ce7a4c755edc5ace9"
}
},
"integrity_after": {
"protocol_sha256": "ff5b33d26103c2a21cec91f91a086ad6420188e99f005cef90f77e3932cfd340",
"packet_sha256": "5b349530c3153c11c7abe0d30dc2073b16091bc82c7d071a6518b4bc4ef53a72",
"packet_manifest_sha256": "968b0159bf001976bf28344d604af7c43136f86df75d430e2734f791010d4701",
"independent_runner_sha256": "01d4f5907e56af599d38ec7b0c873d2940116325d6562cd441efa04dc83fcfca",
"shared_zero_tool_runner_sha256": "f2c66cd60585fcb91291559a2007213e4f4e8cea3f4979b0eb34c82a1bf3d5c0",
"selected_sources_sha256": "dea0e4b71b33afb25fe77f40c7a591f83c453ff436c94b7ddcb0efad7c0fcdba",
"selected_sources": {
"README.md": "1f17edb54748c62426b452d598f6d3779e151a8a132b25b0cf9900899ff625d2",
"SECURITY.md": "d2f29245e3b67aa6e570b269c6121424ebd1a71b1ebb01e69db4415df47038ec",
".claude-plugin/plugin.json": "63a37edb76db4b652b15806f866a2cd335a400acf8455cbc91b3e47430098525",
".claude-plugin/marketplace.json": "89006ac66d5d4eb779dcb07d0f82a18cd9b62c8636bd1b914d99122c0c00831a",
".codex-plugin/plugin.json": "a54a828f5b74f8665ff2c7384aac02accfe50643db6e0a9b84031e8dea43fa43",
"skills/playwright-test-generator/SKILL.md": "f7a29ac29549c4d156caec531111251f3410a0a6a9af06a33c7e2b6622dfd4b0",
"skills/e2e-reviewer/SKILL.md": "e4e1d489c3cc6c54c3a74b825e4e27a9c7ba371c8d9add79ade0f0ec10dbdc2c",
"skills/playwright-debugger/SKILL.md": "2f5bff15ce3219dfb3ce598798b274e3dbe14ab7351009a40ad13ca7677a5d72",
"skills/cypress-debugger/SKILL.md": "f32f38cd2f8b568b35827e4fad99e152067feba1b31fb1403ec16bb78a69cf70",
"skills/e2e-reviewer/references/pattern-reference.md": "4dc1763090cf6738c5166a336e4f4123915efcb5f3f521397e33baa621b83d7e",
"skills/e2e-reviewer/references/verification-rules.md": "09f5998f1262d33affcfb44ce417bd52c9a53909336d505fc085526581361559",
"skills/e2e-reviewer/scripts/scan.sh": "e6e266f291333d5cf6ddc259767db1e3a644963c4657a61762b90f225a60391b",
"skills/playwright-test-generator/scripts/preflight_target.py": "91b277a46fb885b3f6c2b90d31ac72a61fc10a6bd150abbc03eecbe358e9e822",
"skills/playwright-debugger/scripts/read-playwright-artifact.py": "bf16a2c15a5bd4b9710477fd91b4489f02f3aff55d6f98e25568b88d1b7128f6",
"skills/playwright-debugger/scripts/publish-json-report.py": "048e5759d1956e70e4c4ca16bb8543f9edfa0c3861d13ef11e63c24d7fe96ed6",
"skills/playwright-debugger/scripts/download-playwright-report.py": "2570d38939f6a63d0c68d4985861ac4c4710879e13ecb1c19a50a391901d2a61",
"skills/cypress-debugger/scripts/read-cypress-artifact.py": "baca00a68a81a092b7d19a4d0e0535e6aa273c7fd55caa06c721ecc7c638d0ee",
"skills/cypress-debugger/scripts/extract-junit-failures.py": "892a8cb04a84eed8f2c18821623f95e7bd7f1d9e6075cdac05e9f23320556124",
"skills/cypress-debugger/scripts/download-cypress-reports.py": "7b4fc86eb35c3973dc77bb5f71f654faffcfcdfbf1216b7ca749e001258fe091",
"skills/cypress-debugger/scripts/redact_artifact.py": "2e2c85d6219114817f9a0983c3eaf9bde971c589b043fb1997f45d72e08bad67",
"skills/playwright-test-generator/best-practices.md": "b8472249b1f2406e35775eadfe0e973114436844b1117ec804fb5a09817850db",
"skills/playwright-test-generator/code-rules.md": "f1e2567f2938166c6bee7b374ec49f4398c89a32a9143f3d8b8660f0879d3bfe",
"skills/playwright-test-generator/verification-rules.md": "8564baf915821979263cef7b6e588917f33274d9d8bf7826842f192186378216",
"skills/e2e-reviewer/references/grep-patterns.md": "f25037e73e7ae2067f01725cc9c063657c08fc51dffb611c641866ad10317136",
"scripts/ci/ci-local.sh": "85ae26710ecced7f6d46a675ad135cc03ac9ebe408b9929ce7a4c755edc5ace9"
}
},
"review": {
"summary": "The packet defines strong fail-closed scanning, verification, and artifact controls, but the Playwright screenshot workflow lacks the executable trust gate its public contract requires. One reviewer instruction also overstates absence assertions as incapable of failing.",
"scores": {
"semantic_correctness": 89,
"false_positive_control": 95,
"security_trust_boundaries": 80,
"verification_design": 86,
"scope_contract_consistency": 93,
"docs_usability": 85
},
"findings": [
{
"severity": "H",
"category": "security_trust_boundaries",
"file": "skills/playwright-debugger/scripts/read-playwright-artifact.py",
"line": 998,
"title": "No safe reader for Playwright screenshots",
"evidence": "The bundled artifact reader exposes only report and trace modes. In contrast, skills/playwright-debugger/SKILL.md lines 25-33 requires every screenshot and video to pass the bounded reader before a viewer, while line 338 directs users to create and inspect screenshots. The documented safe workflow therefore has no included implementation for standalone Playwright media.",
"recommendation": "Add a descriptor-relative, no-follow Playwright media mode that validates type and size and returns an owner-only immutable snapshot path, or remove standalone screenshot inspection from the workflow."
},
{
"severity": "M",
"category": "semantic_correctness",
"file": "skills/e2e-reviewer/references/pattern-reference.md",
"line": 297,
"title": "Absence assertion mislabeled as unable to fail",
"evidence": "The instruction says to report an unproven absence assertion as an assertion that cannot fail. Such an assertion does fail when the locator resolves to a visible or present element; the actual risk documented earlier is that zero matches also satisfy it, allowing selector rot to pass silently.",
"recommendation": "Describe it as an assertion that can pass without proving the locator ever matched, rather than one that cannot fail."
}
],
"limitations": [
"No commands, tests, benchmarks, holdouts, prior reviews, repository history, or external documentation were used.",
"Runtime success and the behavior of omitted CI helpers cannot be inferred from this packet.",
"The review assesses only included contracts and implementations, not benchmark accuracy or evidence integrity."
],
"verdict": "FAIL"
},
"decision": {
"overall_score": 88.0,
"finding_counts": {
"C": 0,
"H": 1,
"M": 1
},
"checks": {
"overall_score": false,
"dimension_floor": false,
"critical_findings": true,
"high_findings": false,
"model_verdict_matches": true
}
},
"limitations": [
"This is independent context/model-family evidence, not human review.",
"The public packet is not a sealed benchmark or independent ground truth.",
"The Anthropic entries are two models in one provider family, not two independent providers.",
"Excluded scorecards, holdouts, reports, prior reviews, chat conclusions, and git history reduce but cannot prove absence of training-data contamination."
]
}