labels: the review pass, 18 frames, and the one number that matters

7 revisions across 6 frames. All 7 move toward the judge; none move away.

That is the artifact D074 predicted, now measured rather than argued. A set
selected by disagreement and labelled without controls can only raise agreement,
so the size of the rise says nothing: folding these in would lift AC1 by +0.046
(hand count) and +0.052 (manipulation), and raw agreement by about 3.2pp on both
tasks, entirely by construction.

The set was planned as the 18 disagreements only, no control arm, so `report`
now says what that costs rather than printing a bare 33% that reads like
evidence.

Nothing downstream consumes the review pass. The published numbers are unchanged
and the primary labels are untouched.

Author: caiotheodoro

data/labels/caio/review.json

@@ -0,0 +1 @@
+[{"frame_id":"abfa63d5-976a-456a-bd27-ee1b72bf4552","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":1,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:43:16.485486Z","seconds_spent":113},{"frame_id":"3626b5a1-e2cf-4ad3-b187-1c4ff0d652d2","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":2,"manipulation":false,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:43:28.761577Z","seconds_spent":11},{"frame_id":"6c101b44-7b6f-4967-b3dc-bc8958c3d2a2","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":1,"manipulation":true,"edge_case":["partial"],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:43:39.936249Z","seconds_spent":11},{"frame_id":"7801bb78-b711-4859-bf14-eb725fbda0cd","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":2,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:43:44.083315Z","seconds_spent":4},{"frame_id":"715f008d-46ea-429d-9a07-13f2858ebc88","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":2,"manipulation":true,"edge_case":["other-person"],"difficulty":"hard","note":"","labelled_at":"2026-09-05T02:43:58.898520Z","seconds_spent":14},{"frame_id":"54e59589-01c7-491b-af62-7520a919f8b4","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":1,"manipulation":false,"edge_case":["between-actions"],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:44:10.677139Z","seconds_spent":11},{"frame_id":"98b95d40-29ff-4e33-aa90-8516a5422c86","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":2,"manipulation":true,"edge_case":["blur"],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:44:47.840274Z","seconds_spent":37},{"frame_id":"e7cf0a19-464a-4f35-bab2-6c393db75a41","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":2,"manipulation":true,"edge_case":["blur"],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:44:52.674757Z","seconds_spent":4},{"frame_id":"f152de1f-f600-4015-aad4-15500c2c40b4","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":0,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:45:02.632693Z","seconds_spent":9},{"frame_id":"8e715a4b-5942-4cdd-add8-7edad86c0a85","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":2,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:45:14.348386Z","seconds_spent":11},{"frame_id":"48548b29-b04d-4575-9068-d3bee4176b5c","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":1,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:45:22.655534Z","seconds_spent":8},{"frame_id":"17dbd724-f715-4a69-9602-881971246115","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":0,"manipulation":false,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:45:27.774173Z","seconds_spent":5},{"frame_id":"4bd94e12-9d87-4835-9ecd-57f5bdd9c3d4","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":1,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:45:33.605331Z","seconds_spent":5},{"frame_id":"ca7e98e9-ecf4-4916-b75a-e771a3069f63","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":0,"manipulation":false,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:45:40.780271Z","seconds_spent":6},{"frame_id":"d7e64dad-7436-4177-b5d9-1e890f5c697a","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":1,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:45:47.709739Z","seconds_spent":6},{"frame_id":"d8876fdb-eb0a-4ae3-8fce-55f5771969be","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":0,"manipulation":false,"edge_case":["other-person"],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:45:58.615299Z","seconds_spent":10},{"frame_id":"2561b3f1-0ffd-4aab-8677-633f4a15241f","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":1,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:46:03.973395Z","seconds_spent":5},{"frame_id":"5985bb92-b342-4272-8583-5274c744d1a7","rater":"caio","pass":"review","rubric_rev":"1.2.0","hands_visible":1,"manipulation":true,"edge_case":[],"difficulty":"medium","note":"","labelled_at":"2026-09-05T02:46:13.328350Z","seconds_spent":9}]

data/labels/caio/review_set.json

@@ -4,148 +4,76 @@
  "seed": 777,
  "frames": [
   {
-   "frame_id": "d7e64dad-7436-4177-b5d9-1e890f5c697a",
+   "frame_id": "abfa63d5-976a-456a-bd27-ee1b72bf4552",
    "arm": "disagreement"
   },
   {
-   "frame_id": "4862973f-5be8-429d-b4e4-d9f979aa9b43",
-   "arm": "control"
-  },
-  {
-   "frame_id": "43561fa5-f907-41aa-a2cd-537f59e12c55",
-   "arm": "control"
-  },
-  {
-   "frame_id": "d8876fdb-eb0a-4ae3-8fce-55f5771969be",
+   "frame_id": "3626b5a1-e2cf-4ad3-b187-1c4ff0d652d2",
    "arm": "disagreement"
   },
   {
-   "frame_id": "58ac6332-0fdd-46ee-8187-10cce2bbc8b5",
-   "arm": "control"
-  },
-  {
-   "frame_id": "98b95d40-29ff-4e33-aa90-8516a5422c86",
+   "frame_id": "6c101b44-7b6f-4967-b3dc-bc8958c3d2a2",
    "arm": "disagreement"
   },
   {
-   "frame_id": "48548b29-b04d-4575-9068-d3bee4176b5c",
+   "frame_id": "7801bb78-b711-4859-bf14-eb725fbda0cd",
    "arm": "disagreement"
   },
   {
-   "frame_id": "aed8e233-dabc-47cd-afb2-70ac22717802",
-   "arm": "control"
-  },
-  {
-   "frame_id": "f152de1f-f600-4015-aad4-15500c2c40b4",
+   "frame_id": "715f008d-46ea-429d-9a07-13f2858ebc88",
    "arm": "disagreement"
   },
   {
-   "frame_id": "c4fa24d0-f686-4b47-b430-9dab7b802e07",
-   "arm": "control"
-  },
-  {
-   "frame_id": "3626b5a1-e2cf-4ad3-b187-1c4ff0d652d2",
+   "frame_id": "54e59589-01c7-491b-af62-7520a919f8b4",
    "arm": "disagreement"
   },
   {
-   "frame_id": "923b77b6-4bed-42e4-aa98-9335717a4985",
-   "arm": "control"
-  },
-  {
-   "frame_id": "ba2f16e3-1038-47f0-9244-8148acad79ac",
-   "arm": "control"
-  },
-  {
-   "frame_id": "4bd94e12-9d87-4835-9ecd-57f5bdd9c3d4",
+   "frame_id": "98b95d40-29ff-4e33-aa90-8516a5422c86",
    "arm": "disagreement"
   },
   {
-   "frame_id": "ca7e98e9-ecf4-4916-b75a-e771a3069f63",
+   "frame_id": "e7cf0a19-464a-4f35-bab2-6c393db75a41",
    "arm": "disagreement"
   },
   {
-   "frame_id": "0677d71f-a320-49a3-9aba-e2358dd99128",
-   "arm": "control"
-  },
-  {
-   "frame_id": "2561b3f1-0ffd-4aab-8677-633f4a15241f",
+   "frame_id": "f152de1f-f600-4015-aad4-15500c2c40b4",
    "arm": "disagreement"
   },
-  {
-   "frame_id": "9455979e-72fe-4e07-9cf8-c1bce5a8f268",
-   "arm": "control"
-  },
-  {
-   "frame_id": "ebb3db47-db29-4680-b13c-20d9d40dc3bb",
-   "arm": "control"
-  },
   {
    "frame_id": "8e715a4b-5942-4cdd-add8-7edad86c0a85",
    "arm": "disagreement"
   },
   {
-   "frame_id": "fa8e4abb-5622-43a4-ac09-de12420e4b4e",
-   "arm": "control"
-  },
-  {
-   "frame_id": "6c101b44-7b6f-4967-b3dc-bc8958c3d2a2",
+   "frame_id": "48548b29-b04d-4575-9068-d3bee4176b5c",
    "arm": "disagreement"
   },
   {
-   "frame_id": "36ff84fb-27c9-41ed-9303-e4025d6b133b",
-   "arm": "control"
-  },
-  {
-   "frame_id": "abfa63d5-976a-456a-bd27-ee1b72bf4552",
+   "frame_id": "17dbd724-f715-4a69-9602-881971246115",
    "arm": "disagreement"
   },
   {
-   "frame_id": "7801bb78-b711-4859-bf14-eb725fbda0cd",
+   "frame_id": "4bd94e12-9d87-4835-9ecd-57f5bdd9c3d4",
    "arm": "disagreement"
   },
   {
-   "frame_id": "782c1345-7ce8-4a53-99a6-be388a424729",
-   "arm": "control"
-  },
-  {
-   "frame_id": "67114079-6e55-4660-8dd6-9a89b18d9bb0",
-   "arm": "control"
-  },
-  {
-   "frame_id": "5985bb92-b342-4272-8583-5274c744d1a7",
+   "frame_id": "ca7e98e9-ecf4-4916-b75a-e771a3069f63",
    "arm": "disagreement"
   },
   {
-   "frame_id": "715f008d-46ea-429d-9a07-13f2858ebc88",
+   "frame_id": "d7e64dad-7436-4177-b5d9-1e890f5c697a",
    "arm": "disagreement"
   },
   {
-   "frame_id": "ba8acbe6-0786-4af7-ad10-f6ef67055a5f",
-   "arm": "control"
-  },
-  {
-   "frame_id": "a34aa4d2-bd68-440a-ac18-616daa74cac4",
-   "arm": "control"
-  },
-  {
-   "frame_id": "17dbd724-f715-4a69-9602-881971246115",
+   "frame_id": "d8876fdb-eb0a-4ae3-8fce-55f5771969be",
    "arm": "disagreement"
   },
   {
-   "frame_id": "2100ce6e-35c9-42a2-a892-8a9dca3333fb",
-   "arm": "control"
-  },
-  {
-   "frame_id": "e7cf0a19-464a-4f35-bab2-6c393db75a41",
+   "frame_id": "2561b3f1-0ffd-4aab-8677-633f4a15241f",
    "arm": "disagreement"
   },
   {
-   "frame_id": "54e59589-01c7-491b-af62-7520a919f8b4",
+   "frame_id": "5985bb92-b342-4272-8583-5274c744d1a7",
    "arm": "disagreement"
-  },
-  {
-   "frame_id": "bf411945-70e3-4019-b451-4df7b5aeff64",
-   "arm": "control"
   }
  ]
 }

scripts/review_labels_cli.py

@@ -344,6 +344,11 @@ def report(rater: str) -> int:
         print(f"{arm:14s} {s['n']:4d} {s['hands_changed']:7d} {s['manip_changed']:7d} {s['either']:5d} ({rate:.0%})")
 
     d, c = stats["disagreement"], stats["control"]
+    if d["n"] and not c["n"]:
+        print()
+        print("No control arm in this set, so the revision rate above cannot be separated from")
+        print("how often a re-read changes an answer at all. It is a description of what you did,")
+        print("not evidence that the frames were mislabelled.")
     if d["n"] and c["n"]:
         print()
         dr, cr = d["either"] / d["n"], c["either"] / c["n"]