fix(gui): decode inference subprocess output as UTF-8 (#2744) (#2801)

The GUI inference runner read the subprocess output in text mode without
specifying an encoding, so Python used the locale default — cp1252 on
Windows. When sleap-nn emitted any non-cp1252 byte (e.g. the UTF-8
box-drawing glyphs in a `rich`-rendered traceback or progress bar), the
reader crashed with `UnicodeDecodeError: 'charmap' codec can't decode
byte 0x90`, masking the real underlying error from the user.

Force `encoding="utf-8", errors="replace"` on the `Popen` reader (and on
the sibling raw-bytes `.decode()` path) so the real subprocess output —
including error tracebacks — surfaces in the GUI log on all platforms.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: gitttt-1234

sleap/gui/learning/runners.py

@@ -239,6 +239,15 @@ def _run_inference_item(
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,  # Merge stderr into stdout
             text=True,  # Text mode (required for line buffering)
+            # Force UTF-8 decoding of the child's output. Without this, text mode
+            # decodes using the locale default, which is cp1252 on Windows, so any
+            # non-cp1252 byte in the subprocess output (e.g. the UTF-8 box-drawing
+            # glyphs in a `rich`-rendered traceback or progress bar) crashes the
+            # reader with `UnicodeDecodeError: 'charmap' codec can't decode byte ...`,
+            # masking the real error. `errors="replace"` keeps reading even if the
+            # child emits a stray non-UTF-8 byte. (gh discussion #2744)
+            encoding="utf-8",
+            errors="replace",
             bufsize=1,  # Line buffered
             env=env,
         ) as proc:
@@ -700,7 +709,9 @@ def predict_subprocess(
             while proc.poll() is None:
                 # Read line.
                 line = proc.stdout.readline()
-                line = line.decode().rstrip()
+                # Decode as UTF-8 with replacement so a stray non-UTF-8 byte in
+                # the subprocess output can't crash the reader (see #2744).
+                line = line.decode("utf-8", errors="replace").rstrip()
 
                 is_json = False
                 if line.startswith("{"):

tests/gui/learning/test_runners.py

@@ -0,0 +1,52 @@
+"""Tests for inference/training subprocess runners."""
+
+import sys
+
+from unittest.mock import MagicMock
+
+from sleap.gui.learning.runners import InferenceWorker
+
+
+def test_inference_worker_reads_non_utf8_subprocess_output(qtbot, tmp_path):
+    """Reading inference subprocess output must not crash on a stray non-UTF-8 byte.
+
+    Regression test for gh discussion #2744 / sleap-nn #655: on Windows the GUI
+    decoded the inference subprocess output using the cp1252 locale default, so a
+    byte like ``0x90`` (e.g. from a ``rich``-rendered traceback or progress bar)
+    raised ``UnicodeDecodeError: 'charmap' codec can't decode byte 0x90`` and
+    masked the real error. A lone ``0x90`` is also invalid UTF-8, so it reproduces
+    the crash on any platform when the reader does not pass ``errors="replace"``.
+    """
+    # Child process emits a lone 0x90 byte (invalid UTF-8 / cp1252-undefined),
+    # stays alive briefly so the parent loop reads + decodes it, then exits
+    # non-zero so the success/load-slp branch is skipped.
+    script = (
+        "import sys, time; "
+        "sys.stdout.buffer.write(b'progress \\x90 line\\n'); "
+        "sys.stdout.flush(); "
+        "time.sleep(0.5); "
+        "sys.exit(1)"
+    )
+    output_path = str(tmp_path / "out.slp")
+
+    task = MagicMock()
+    task.make_predict_cli_call.return_value = (
+        [sys.executable, "-c", script],
+        output_path,
+    )
+
+    items = MagicMock()
+    items.total_frame_count = 0
+
+    worker = InferenceWorker(task, items)
+
+    logged = []
+    worker.logOutput.connect(logged.append)
+
+    # Must not raise UnicodeDecodeError while reading the subprocess output.
+    out_path, ret = worker._run_inference_item(MagicMock(), 0, 1)
+
+    assert out_path == output_path
+    assert ret == 1  # non-zero exit propagated; no decode crash
+    # The malformed line was still captured (with the bad byte replaced).
+    assert any("progress" in line and "line" in line for line in logged)