Skip to content

Commit ab7f54b

Browse files
committed
fix: decode subprocess output with errors="replace" to keep the reader thread alive
The subprocess stdout/stderr are decoded with a strict UTF-8 handler, so a single non-UTF-8 byte (e.g. cp1252 output from a DCC batch renderer) raises UnicodeDecodeError in StreamLogger.run. That exception kills the stdout reader thread; the pipe then stops being drained and the subprocess can deadlock. Pass errors="replace" to the Popen call so undecodable bytes are replaced instead of raising. Add a regression test that a subprocess emitting a non-UTF-8 byte does not kill the reader thread, and update the existing Popen argument assertions. Fixes: #274 Signed-off-by: Takumi Koshii <koshii.takumi@classmethod.jp>
1 parent 7084dd7 commit ab7f54b

3 files changed

Lines changed: 41 additions & 0 deletions

File tree

‎src/openjd/adaptor_runtime/process/_logging_subprocess.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,13 @@ def __init__(
6464
stderr=subprocess.PIPE,
6565
stdout=subprocess.PIPE,
6666
encoding=encoding,
67+
# Decode the subprocess output leniently. DCC batch processes may emit
68+
# locale-encoded (non-UTF-8) bytes on stdout/stderr; with the default strict
69+
# error handling a single undecodable byte raises UnicodeDecodeError in the
70+
# stream-reader thread, killing it. Once the reader thread is gone the pipe is
71+
# no longer drained, the subprocess blocks on a full stdout pipe, and the job
72+
# deadlocks. Replacing undecodable bytes keeps the reader alive.
73+
errors="replace",
6774
cwd=startup_directory,
6875
)
6976
if OSName.is_windows(): # pragma: is-posix

‎test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,37 @@ def test_log_levels(self, log_level: int, caplog):
151151

152152
assert any(r.message == message and r.levelno == _STDERR_LEVEL for r in records)
153153

154+
@pytest.mark.timeout(10)
155+
def test_non_utf8_output_does_not_kill_reader(self, caplog):
156+
"""
157+
A subprocess that writes a non-UTF-8 (locale-encoded) byte to stdout must not kill the
158+
stream-reader thread. If the reader thread dies, lines emitted after the undecodable byte
159+
are never logged and the stdout pipe stops being drained, which can deadlock the subprocess.
160+
This is a regression test for that failure mode; the undecodable byte must be tolerated and
161+
subsequent output must still be logged.
162+
"""
163+
# GIVEN
164+
caplog.set_level(_STDOUT_LEVEL)
165+
# 0xc7 is "Ç" in cp1252 and is not a valid standalone UTF-8 byte.
166+
script = (
167+
"import sys; "
168+
"sys.stdout.buffer.write(b'before\\n'); "
169+
"sys.stdout.buffer.write(b'bad \\xc7 byte\\n'); "
170+
"sys.stdout.buffer.write(b'after\\n'); "
171+
"sys.stdout.buffer.flush()"
172+
)
173+
174+
# WHEN
175+
p = LoggingSubprocess(args=[sys.executable, "-c", script])
176+
p.wait()
177+
p._cleanup_io_threads()
178+
179+
# THEN
180+
# "after" is only logged if the reader thread survived the undecodable byte.
181+
assert "after" in caplog.text
182+
# The undecodable byte is replaced rather than raising.
183+
assert "�" in caplog.text
184+
154185
def test_executable_not_found(self):
155186
"""When calling LoggingSubprocess with a missing executable, FileNotFoundError will be raised"""
156187
args = ["missing_executable"]

‎test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ def test_process_creation(self, mock_popen: mock.Mock, mock_stream_logger: mock.
6969
popen_params = dict(
7070
args=args,
7171
encoding="utf-8",
72+
errors="replace",
7273
stdin=subprocess.PIPE,
7374
stdout=subprocess.PIPE,
7475
stderr=subprocess.PIPE,
@@ -482,6 +483,7 @@ def test_startup_directory_default(self, mock_popen_autospec: mock.Mock):
482483
stdout=subprocess.PIPE,
483484
stderr=subprocess.PIPE,
484485
encoding="utf-8",
486+
errors="replace",
485487
cwd=None,
486488
)
487489
if OSName.is_windows():
@@ -504,6 +506,7 @@ def test_start_directory(self, mock_popen_autospec: mock.Mock):
504506
stdout=subprocess.PIPE,
505507
stderr=subprocess.PIPE,
506508
encoding="utf-8",
509+
errors="replace",
507510
cwd="startup_dir",
508511
)
509512
if OSName.is_windows():

0 commit comments

Comments
 (0)