Skip to content

Commit c2791d0

Browse files
committed
feat(util): replay parallel task output as log records
- what a task printed now arrives as INFO records on a dedicated osw.parallel.output logger, one record per line - keeps the library off the process's real stdout, which an MCP stdio server needs clear for its JSON-RPC channel - BEHAVIOUR CHANGE: a direct parallelize caller who read a task's printed output from captured stdout now gets nothing there - the ordering guarantee is unchanged: items follow the iterable
1 parent 8d24914 commit c2791d0

4 files changed

Lines changed: 115 additions & 41 deletions

File tree

‎docs/get-started.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,16 @@ One case osw cannot detect is a handler added to the `osw` logger itself, since
117117
that is indistinguishable from one of its own. Call `disable_logging()` first if
118118
you do that.
119119

120+
### Output from parallel batches
121+
122+
Several osw calls process their input in parallel. What such a task prints, as
123+
opposed to logs, is collected while the batch runs and replayed afterwards, so
124+
that concurrent writing cannot garble the progress bar. That text arrives on the
125+
`osw.parallel.output` logger, one record per line, at INFO. Give that logger its
126+
own level or handler to keep the output of third-party code apart from osw's own
127+
records. Most osw calls replay it only when you pass `debug=True`; `copy_pages`
128+
always replays.
129+
120130
## Examples and tutorials
121131

122132
- Runnable scripts in

‎src/osw/utils/util.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@
1818

1919
_logger = logging.getLogger(__name__)
2020

21+
# What a parallelized task printed rather than logged. It gets a name of its own
22+
# so a caller can filter the output of foreign code apart from osw's own records,
23+
# and so the name describes what the records carry rather than which function
24+
# replayed them.
25+
TASK_OUTPUT_LOGGER = "osw.parallel.output"
26+
2127
# dask.config.set(scheduler="threads")
2228
# with stdio_proxy.redirect_stdout(sys.stdout):
2329
# "processes" scheduler leads to no messages ending up in the buffer / no flushing
@@ -413,11 +419,14 @@ def parallelize(
413419
The iterable, who's items are to be passed as singles to the function.
414420
flush_at_end:
415421
If True, what the tasks print and log is written out in the order of
416-
``iterable`` once the batch has finished. If False, it is discarded,
417-
except for log records at WARNING and above. Those report a problem
418-
rather than progress, so they are written out either way. Either way
419-
the output is captured while the tasks run, so that concurrent writing
420-
cannot garble the progress bar.
422+
``iterable`` once the batch has finished. What a task printed is
423+
replayed one line at a time as INFO records on the ``osw.parallel.output``
424+
logger, so it reaches the caller's handlers like everything else rather
425+
than going straight to stdout. If False, both are discarded, except for
426+
log records at WARNING and above. Those report a problem rather than
427+
progress, so they are written out either way. Either way the output is
428+
captured while the tasks run, so that concurrent writing cannot garble
429+
the progress bar.
421430
progress_bar:
422431
If True, a progress bar will be displayed.
423432
return_exceptions:
@@ -506,9 +515,14 @@ async def _capture(task):
506515
osw_logger.propagate = saved_propagate
507516
# in the finally block, so a failed batch still reports what its
508517
# tasks had to say
518+
task_output = logging.getLogger(TASK_OUTPUT_LOGGER)
509519
for output, task_records in zip(outputs, records):
510-
if flush_at_end and output:
511-
print(output, end="")
520+
if flush_at_end:
521+
# One record per line, so each carries its own level and name
522+
# instead of one prefix followed by a block of text
523+
for line in output.splitlines():
524+
if line:
525+
task_output.info(line)
512526
for record in task_records:
513527
# A warning reports a problem, not progress, so a batch asked
514528
# to stay quiet still passes it on. Discarding it would hide

‎tests/test_logging_setup.py‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@
1212
import pytest
1313

1414
import osw
15-
from osw.utils.util import ThreadRoutedLogHandler, handler_chain, parallelize
15+
from osw.utils.util import (
16+
TASK_OUTPUT_LOGGER,
17+
ThreadRoutedLogHandler,
18+
handler_chain,
19+
parallelize,
20+
)
1621

1722

1823
class ListHandler(logging.Handler):
@@ -403,3 +408,45 @@ def test_a_failed_batch_still_replays_and_restores(collected, osw_logger):
403408
assert collected.messages(WORKER) == ["about to fail on 7"]
404409
assert collected in osw_logger.handlers
405410
assert not [h for h in osw_logger.handlers if isinstance(h, ThreadRoutedLogHandler)]
411+
412+
413+
def _print_item(item):
414+
print(f"printed {item}")
415+
return item
416+
417+
418+
def _print_two_lines(item):
419+
print("first line")
420+
print("second line")
421+
return item
422+
423+
424+
def test_task_output_is_logged_on_its_own_logger(collected):
425+
"""Captured print output carries its own logger name, distinct from
426+
osw.utils.util where parallelize itself logs from, so a caller can filter
427+
it apart from osw's own records."""
428+
parallelize(_print_item, [1], flush_at_end=True, progress_bar=False)
429+
430+
record = next(r for r in collected.records if "printed 1" in r.getMessage())
431+
assert record.name == TASK_OUTPUT_LOGGER
432+
assert record.name != "osw.utils.util"
433+
434+
435+
def test_task_output_is_split_into_one_record_per_line(collected):
436+
"""A block of printed text arrives as a run of lines, each with its own
437+
record, rather than one record holding the whole block."""
438+
parallelize(_print_two_lines, [1], flush_at_end=True, progress_bar=False)
439+
440+
messages = [
441+
r.getMessage() for r in collected.records if r.name == TASK_OUTPUT_LOGGER
442+
]
443+
assert messages == ["first line", "second line"]
444+
445+
446+
def test_task_output_no_longer_reaches_real_stdout(capsys):
447+
"""Printed output is replayed through logging now, not written back to the
448+
process's own stdout, which an MCP stdio server needs kept clear for the
449+
JSON-RPC channel it carries."""
450+
parallelize(_print_item, [1], flush_at_end=True, progress_bar=False)
451+
452+
assert "printed 1" not in capsys.readouterr().out

‎tests/utils/parallelize_test.py‎

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,13 @@
1010
"""
1111

1212
import asyncio
13+
import logging
1314
import sys
1415
import threading
1516

1617
import pytest
1718

18-
from osw.utils.util import ThreadRoutedStdout, parallelize
19+
from osw.utils.util import TASK_OUTPUT_LOGGER, ThreadRoutedStdout, parallelize
1920

2021
# Generous enough that a loaded CI machine still passes, short enough that a
2122
# serialized execution trips it instead of hanging the suite.
@@ -123,19 +124,23 @@ def _print_then_return(item):
123124
return item
124125

125126

126-
def test_flush_at_end_prints_what_the_tasks_printed(capsys):
127+
def test_flush_at_end_logs_what_the_tasks_printed(caplog):
128+
caplog.set_level(logging.INFO, logger=TASK_OUTPUT_LOGGER)
129+
127130
parallelize(_print_then_return, [1, 2, 3], flush_at_end=True)
128131

129-
out = capsys.readouterr().out
130-
assert "handled 1" in out
131-
assert "handled 2" in out
132-
assert "handled 3" in out
132+
messages = [r.message for r in caplog.records if r.name == TASK_OUTPUT_LOGGER]
133+
assert "handled 1" in messages
134+
assert "handled 2" in messages
135+
assert "handled 3" in messages
133136

134137

135-
def test_flush_at_end_replays_in_input_order(capsys):
138+
def test_flush_at_end_replays_in_input_order(caplog):
136139
"""Output follows the iterable, not the order the tasks happened to finish."""
137140
import time
138141

142+
caplog.set_level(logging.INFO, logger=TASK_OUTPUT_LOGGER)
143+
139144
delays = {1: 0.05, 2: 0.0, 3: 0.03}
140145

141146
def sleep_then_print(item):
@@ -144,49 +149,47 @@ def sleep_then_print(item):
144149

145150
parallelize(sleep_then_print, [1, 2, 3], flush_at_end=True)
146151

147-
out = capsys.readouterr().out
148-
assert out.index("handled 1") < out.index("handled 2") < out.index("handled 3")
152+
messages = [r.message for r in caplog.records if r.name == TASK_OUTPUT_LOGGER]
153+
assert messages.index("handled 1") < messages.index("handled 2")
154+
assert messages.index("handled 2") < messages.index("handled 3")
149155

150156

151-
def test_flush_at_end_defers_until_the_batch_is_done():
152-
"""Nothing reaches the real stdout while the tasks are still running."""
153-
written = []
157+
def test_flush_at_end_defers_until_the_batch_is_done(caplog):
158+
"""Nothing reaches the osw.parallel.output logger while the tasks are still
159+
running."""
154160
snapshots = []
155161
barrier = threading.Barrier(3, timeout=BARRIER_TIMEOUT)
156162

157-
class _Recorder:
158-
def write(self, message):
159-
written.append(message)
160-
return len(message)
161-
162-
def flush(self):
163-
pass
163+
caplog.set_level(logging.INFO, logger=TASK_OUTPUT_LOGGER)
164164

165165
def print_then_look(item):
166166
print(f"handled {item}")
167167
barrier.wait() # every task has printed by the time this releases
168-
snapshots.append("".join(written))
168+
snapshots.append([
169+
r.message for r in caplog.records if r.name == TASK_OUTPUT_LOGGER
170+
])
169171

170-
original = sys.stdout
171-
sys.stdout = _Recorder()
172-
try:
173-
parallelize(print_then_look, [1, 2, 3], flush_at_end=True, progress_bar=False)
174-
finally:
175-
sys.stdout = original
172+
parallelize(print_then_look, [1, 2, 3], flush_at_end=True, progress_bar=False)
173+
174+
# mid-flight the logger had seen none of it, afterwards it has all of it
175+
assert not any(
176+
any("handled" in message for message in snapshot) for snapshot in snapshots
177+
)
178+
final = [r.message for r in caplog.records if r.name == TASK_OUTPUT_LOGGER]
179+
assert sum("handled" in message for message in final) == 3
176180

177-
# mid-flight the real stream had seen none of it, afterwards it has all of it
178-
assert not any("handled" in snapshot for snapshot in snapshots)
179-
assert "".join(written).count("handled") == 3
180181

182+
def test_task_output_is_suppressed_without_flush_at_end(caplog):
183+
caplog.set_level(logging.INFO, logger=TASK_OUTPUT_LOGGER)
181184

182-
def test_task_output_is_suppressed_without_flush_at_end(capsys):
183185
parallelize(_print_then_return, [1, 2, 3])
184186

185-
assert "handled" not in capsys.readouterr().out
187+
assert not any("handled" in record.message for record in caplog.records)
186188

187189

188-
def test_flush_at_end_reports_even_when_the_batch_fails(capsys):
190+
def test_flush_at_end_reports_even_when_the_batch_fails(caplog):
189191
"""A failing item must not swallow the diagnostics of the others."""
192+
caplog.set_level(logging.INFO, logger=TASK_OUTPUT_LOGGER)
190193

191194
def print_then_fail(item):
192195
print(f"handled {item}")
@@ -197,7 +200,7 @@ def print_then_fail(item):
197200
with pytest.raises(ValueError, match="boom"):
198201
parallelize(print_then_fail, [1, 2, 3], flush_at_end=True)
199202

200-
assert "handled 1" in capsys.readouterr().out
203+
assert any("handled 1" in record.message for record in caplog.records)
201204

202205

203206
def test_stdout_is_restored_afterwards():

0 commit comments

Comments
 (0)