test: make OLED ceremony evidence semantic

Author: BitHighlander

keepkeylib/client.py

@@ -432,13 +432,13 @@ def set_mnemonic(self, mnemonic):
 
     def call_raw(self, msg):
 
-        # Screenshot capture disabled in call_raw (captures idle screens, adds latency).
-        # Real confirmation screenshots are captured in callback_ButtonRequest instead.
-        # Exception: capture on Failure (rejection screens like invalid BIP-39 word).
+        # Screenshot capture is disabled in call_raw: a wire Failure is often
+        # emitted after the UI has already returned to the lock/home screen.
+        # Treating that framebuffer as operation evidence created convincing
+        # but unrelated blank/lock frames. Tests that claim a visible rejection
+        # capture it explicitly at the point the firmware renders it.
 
         resp = super(DebugLinkMixin, self).call_raw(msg)
-        if isinstance(resp, proto.Failure):
-            self._capture_oled()
         self._check_request(resp)
         return resp
 
@@ -501,6 +501,35 @@ def _capture_oled(self, layout=None):
             traceback.print_exc(file=sys.stderr)
             raise
 
+    def _capture_oled_after_animation(self, seconds, required_region=None):
+        """Capture the completed PIN/cipher frame, not its initial blank state."""
+        if not SCREENSHOT:
+            return
+        # The emulator's PIN/recovery loop blocks while waiting for host
+        # input. Wall-clock sleep alone therefore does not repaint: each
+        # DebugLink request wakes the loop for one 20 ms animation tick.
+        # Drive every required tick and retain the layout from the final poll.
+        layout = None
+        for _tick in range(int(seconds / 0.020) + 2):
+            time.sleep(0.025)
+            layout = self.debug.read_layout()
+        if required_region:
+            x0, x1, y0, y1 = required_region
+            lit = 0
+            for y in range(y0, y1):
+                for x in range(x0, x1):
+                    byte_index = x + (y // 8) * 256
+                    value = layout[byte_index]
+                    if not isinstance(value, int):
+                        value = ord(value)
+                    lit += (value >> (y % 8)) & 1
+            area = (x1 - x0) * (y1 - y0)
+            if lit < 32 or area - lit < 32:
+                raise RuntimeError(
+                    "animated OLED evidence lacks grid contrast "
+                    "(%d lit of %d pixels)" % (lit, area))
+        self._capture_oled(layout)
+
     def callback_ButtonRequest(self, msg):
         if self.verbose:
             log("ButtonRequest code: " + get_buttonrequest_value(msg.code))
@@ -520,6 +549,9 @@ def callback_ButtonRequest(self, msg):
         return proto.ButtonAck()
 
     def callback_PinMatrixRequest(self, msg):
+        # Firmware animates the randomized grid for PIN_MAX_ANIMATION_MS
+        # (1000 ms). Sampling immediately captures only the prompt/blank mask.
+        self._capture_oled_after_animation(1.05, (192, 256, 0, 64))
         if self.pin_correct:
             pin = self.debug.read_pin_encoded()
         else:

scripts/generate-test-report.py

@@ -180,8 +180,8 @@ def finish(self):
         self._flush()
 
 def _lookup(results, mod, meth):
-    """Look up test result by module::method (precise), then bare method (fallback)."""
-    return results.get(f'{mod}::{meth}') or results.get(meth) or ''
+    """Look up only the exact catalog module::method identity."""
+    return results.get(f'{mod}::{meth}') or ''
 
 def ver_t(s): return tuple(int(x) for x in s.replace('v','').split('.')[:3])
 def ver_ge(a, b): return ver_t(a) >= ver_t(b)
@@ -205,8 +205,7 @@ def detect_fw():
     except: return None
 
 def parse_junit(path):
-    """Parse junit XML for pass/fail. Returns dict keyed by 'module::method' (precise)
-    and 'method' (fallback). Module is extracted from classname: tests.test_msg_foo.TestBar → test_msg_foo."""
+    """Parse JUnit status under exact module::method and native-suite keys."""
     if not path or not os.path.exists(path): return {}
     import xml.etree.ElementTree as ET
     results = {}
@@ -217,12 +216,15 @@ def parse_junit(path):
         elif tc.find('error') is not None: status = 'error'
         elif tc.find('skipped') is not None: status = 'skip'
         else: status = 'pass'
-        # Extract module from classname: tests.test_msg_foo.TestBar → test_msg_foo
+        # Extract the test module from a pytest classname such as
+        # tests.test_protection_levels.TestProtectionLevels. Exact lookup is
+        # safe across every test_* module and avoids collision-prone method
+        # fallback.
         mod = ''
         if cls:
             parts = cls.split('.')
             for p in parts:
-                if p.startswith('test_msg_') or p.startswith('test_sign_') or p.startswith('test_verify_'):
+                if p.startswith('test_'):
                     mod = p
                     break
             results[f'{cls}.{name}'] = status
@@ -233,9 +235,6 @@ def parse_junit(path):
         # Key by module::method (disambiguates collisions like test_sign_btc_eth_swap)
         if mod:
             results[f'{mod}::{name}'] = status
-        # Bare method fallback -- only set if no collision
-        if name not in results or status == 'pass':
-            results[name] = status
     return results
 
 
@@ -379,6 +378,10 @@ def junit_reconciliation(path):
         ('test_msg_resetdevice', 'test_reset_device_pin'),
         ('test_msg_resetdevice', 'test_reset_device_18_words'),
         ('test_msg_resetdevice', 'test_reset_device_24_words'),
+        ('test_msg_recoverydevice_cipher', 'test_nopin_nopassphrase'),
+        ('test_msg_recoverydevice_cipher', 'test_pin_passphrase'),
+        ('test_msg_changepin', 'test_set_pin'),
+        ('test_msg_ethereum_signtx', 'test_ethereum_blind_sign_allowed'),
         ('test_msg_ethereum_signtx', 'test_ethereum_blind_sign_blocked'),
         ('BodyFits',
          'ConstantPowerSeedRowsAreCompleteAndPagedAtRowBoundaries'),
@@ -607,6 +610,8 @@ def junit_reconciliation(path):
           'Generate 12-word seed',
           'Device generates 128 bits of entropy from its RNG, combines it with host entropy, '
           'and displays the resulting 12-word BIP-39 mnemonic on every physical OLED page. '
+          'Each 256x64 image is one physical frame; its opposing-polarity halves are the '
+          'intentional constant-power layout, not a stitched composite. '
           'Words are NEVER sent over the normal host transport. '
           'User writes them down as their backup.',
           ['Seed word display']),
@@ -623,7 +628,8 @@ def junit_reconciliation(path):
          ('C3', 'test_msg_resetdevice', 'test_reset_device_pin',
           'Generate seed with PIN',
           'Same as C2 but also sets a PIN. PIN is entered twice for confirmation via the '
-          'randomized 3x3 grid on OLED. Verifies PIN is stored and required for subsequent operations.',
+          'fully animated randomized 3x3 grid on OLED. Verifies PIN is stored and required for '
+          'subsequent operations.',
           ['PIN entry grid']),
          ('C4', 'test_msg_resetdevice', 'test_failed_pin',
           'PIN mismatch rejects setup',
@@ -655,14 +661,16 @@ def junit_reconciliation(path):
           []),
          ('C10', 'test_msg_recoverydevice_cipher', 'test_nopin_nopassphrase',
           'Cipher recovery (no PIN)',
-          'Recovery via scrambled keyboard on OLED. The letter grid is randomized per-character, '
+          'Recovery via the fully animated scrambled keyboard on OLED. The letter grid is '
+          'randomized per-character, '
           'so even a compromised host cannot determine which letters the user selected. After all '
-          'words are entered, device verifies BIP-39 checksum and reconstructs the seed.',
+          'words are entered, the terminal Success proves checksum validation and seed '
+          'reconstruction; cipher entry itself may end on an auto-completed word prefix.',
           ['Cipher grid on OLED']),
          ('C11', 'test_msg_recoverydevice_cipher', 'test_pin_passphrase',
           'Cipher recovery with PIN + passphrase',
-          'Same recovery flow as C10 but also sets PIN and enables passphrase protection during '
-          'the recovery process.',
+          'Same recovery flow as C10 but also captures the completed PIN grids and enables '
+          'passphrase protection as a stored setting; no passphrase value is entered during recovery.',
           ['Cipher + PIN entry']),
          ('C12', 'test_msg_recoverydevice_cipher', 'test_character_fail',
           'Invalid character rejection',
@@ -1117,7 +1125,7 @@ def junit_reconciliation(path):
           'Blind sign permitted (AdvancedMode ON)',
           'Contract data with AdvancedMode enabled. Device allows signing only after opaque-policy '
           'and complete-calldata hash disclosure.',
-          []),
+          ['Opaque policy warning', 'Complete calldata hash approval']),
          ('V9', 'test_msg_ethereum_signtx', 'test_ethereum_blind_sign_blocked',
           'Opaque contract data blocked (AdvancedMode OFF)',
           'On 7.14.2 the same arbitrary calldata is refused by policy before any signing approval. '
@@ -1513,6 +1521,7 @@ def render(output_path, fw_version, results, screenshot_dir=None,
     pb.text(7, f'Generator blob SHA: {provenance.get("generator_blob_sha", "MISSING")}')
     pb.text(7, f'Generator SHA-256: {provenance.get("generator_sha256", "MISSING")}')
     pb.text(7, f'ARM manifest SHA-256: {provenance.get("arm_manifest_sha256", "MISSING")}')
+    pb.text(7, f'Report controls SHA-256: {provenance.get("report_controls_sha256", "MISSING")}')
     pb.text(7, f'Build: {provenance.get("build_label", "MISSING")}')
     for line in _w(f'CI run: {provenance.get("run_url", "MISSING")}', 88):
         pb.text(7, line)
@@ -1950,6 +1959,8 @@ def main():
     p.add_argument('--run-url', default=None, help='immutable CI run URL')
     p.add_argument('--arm-manifest-sha256', default=None,
                    help='SHA-256 of the exact-head ARM binary manifest')
+    p.add_argument('--report-controls-sha256', default=None,
+                   help='SHA-256 of the passing report mutation controls')
     p.add_argument('--completeness-output', default=None,
                    help='write machine-readable catalog/frame completeness JSON')
     p.add_argument('--required-tests-manifest', default=None,
@@ -2024,6 +2035,7 @@ def main():
         'build_label': args.build_label,
         'run_url': args.run_url,
         'arm_manifest_sha256': args.arm_manifest_sha256,
+        'report_controls_sha256': args.report_controls_sha256,
     }
     missing_provenance = [k for k, value in required_provenance.items() if not value]
     if missing_provenance:
@@ -2053,6 +2065,12 @@ def main():
         print('ERROR: ARM manifest SHA-256 is not exact: %s' %
               args.arm_manifest_sha256, file=sys.stderr)
         sys.exit(2)
+    if (len(args.report_controls_sha256) != 64 or
+            any(c not in '0123456789abcdef'
+                for c in args.report_controls_sha256.lower())):
+        print('ERROR: report controls SHA-256 is not exact: %s' %
+              args.report_controls_sha256, file=sys.stderr)
+        sys.exit(2)
     catalog_errors = catalog_integrity(fw, derived_sources)
     if catalog_errors:
         print('ERROR: release evidence catalog is inconsistent:', file=sys.stderr)

tests/test_msg_changepin.py

@@ -46,11 +46,13 @@ def test_set_pin(self):
 
         # Send the PIN for first time
         self.assertIsInstance(ret, proto.PinMatrixRequest)
+        self.client._capture_oled_after_animation(1.05, (192, 256, 0, 64))
         pin_encoded = self.client.debug.encode_pin(self.pin6)
         ret = self.client.call_raw(proto.PinMatrixAck(pin=pin_encoded))
 
         # Send the PIN for second time
         self.assertIsInstance(ret, proto.PinMatrixRequest)
+        self.client._capture_oled_after_animation(1.05, (192, 256, 0, 64))
         pin_encoded = self.client.debug.encode_pin(self.pin6)
         ret = self.client.call_raw(proto.PinMatrixAck(pin=pin_encoded))
 

tests/test_msg_recoverydevice_cipher.py

@@ -49,7 +49,7 @@ def test_pin_passphrase(self):
 
         # Reminder UI
         assert isinstance(ret, proto.ButtonRequest)
-        self.client._capture_oled()
+        self.client._capture_oled_after_animation(0.35, (76, 256, 0, 64))
         self.client.debug.press_yes()
         ret = self.client.call_raw(proto.ButtonAck())
 
@@ -58,7 +58,8 @@ def test_pin_passphrase(self):
         for index, word in enumerate(mnemonic_words):
             for character in word:
                 self.assertIsInstance(ret, proto.CharacterRequest)
-                self.client._capture_oled()
+                self.client._capture_oled_after_animation(
+                    0.35, (76, 256, 0, 64))
                 cipher = self.client.debug.read_recovery_cipher()
 
                 encoded_character = cipher[ord(character) - 97]
@@ -109,7 +110,7 @@ def test_nopin_nopassphrase(self):
 
         # Reminder UI
         assert isinstance(ret, proto.ButtonRequest)
-        self.client._capture_oled()
+        self.client._capture_oled_after_animation(0.35, (76, 256, 0, 64))
         self.client.debug.press_yes()
         ret = self.client.call_raw(proto.ButtonAck())
 
@@ -118,7 +119,8 @@ def test_nopin_nopassphrase(self):
         for index, word in enumerate(mnemonic_words):
             for character in word:
                 self.assertIsInstance(ret, proto.CharacterRequest)
-                self.client._capture_oled()
+                self.client._capture_oled_after_animation(
+                    0.35, (76, 256, 0, 64))
                 cipher = self.client.debug.read_recovery_cipher()
 
                 encoded_character = cipher[ord(character) - 97]

tests/test_msg_resetdevice.py

@@ -143,13 +143,13 @@ def test_reset_device_pin(self):
             ret = self.client.call_raw(proto.ButtonAck())
 
         self.assertIsInstance(ret, proto.PinMatrixRequest)
-        self.client._capture_oled()
+        self.client._capture_oled_after_animation(1.05, (192, 256, 0, 64))
 
         # Enter PIN for first time
         pin_encoded = self.client.debug.encode_pin('654')
         ret = self.client.call_raw(proto.PinMatrixAck(pin=pin_encoded))
         self.assertIsInstance(ret, proto.PinMatrixRequest)
-        self.client._capture_oled()
+        self.client._capture_oled_after_animation(1.05, (192, 256, 0, 64))
 
         # Enter PIN for second time
         pin_encoded = self.client.debug.encode_pin('654')
@@ -224,13 +224,13 @@ def test_failed_pin(self):
             ret = self.client.call_raw(proto.ButtonAck())
 
         self.assertIsInstance(ret, proto.PinMatrixRequest)
-        self.client._capture_oled()
+        self.client._capture_oled_after_animation(1.05, (192, 256, 0, 64))
 
         # Enter PIN for first time
         pin_encoded = self.client.debug.encode_pin(self.pin4)
         ret = self.client.call_raw(proto.PinMatrixAck(pin=pin_encoded))
         self.assertIsInstance(ret, proto.PinMatrixRequest)
-        self.client._capture_oled()
+        self.client._capture_oled_after_animation(1.05, (192, 256, 0, 64))
 
         # Enter PIN for second time
         pin_encoded = self.client.debug.encode_pin(self.pin6)