11import asyncio
22import functools
3+ import logging
34import sys
45import threading
56from asyncio import Queue
@@ -309,18 +310,85 @@ def __getattr__(self, name):
309310 return getattr (self .stream , name )
310311
311312
313+ class ThreadRoutedLogHandler (logging .Handler ):
314+ """A log handler that routes each worker thread's records to its own buffer
315+
316+ The counterpart of ThreadRoutedStdout for the modules that log rather than
317+ print. Installed on the 'osw' logger for the duration of a parallelize()
318+ call, in place of the handlers a record would otherwise reach. Threads that
319+ have not registered a buffer, the calling thread among them, are forwarded
320+ to those handlers unchanged.
321+
322+ Routing has to happen at the handler, not at a filter on the 'osw' logger. A
323+ logger's filters only run for records logged on that logger itself, so they
324+ would never see the records from osw.wtsite and the other child loggers.
325+ """
326+
327+ def __init__ (self , handlers : List [logging .Handler ]):
328+ super ().__init__ ()
329+ self .handlers = handlers
330+ self ._local = threading .local ()
331+
332+ def register (self ) -> List [logging .LogRecord ]:
333+ """Gives the calling thread its own buffer and returns it"""
334+ buffer = []
335+ self ._local .buffer = buffer
336+ return buffer
337+
338+ def unregister (self ):
339+ """Sends the calling thread's records back to the wrapped handlers"""
340+ self ._local .buffer = None
341+
342+ def emit (self , record : logging .LogRecord ):
343+ buffer = getattr (self ._local , "buffer" , None )
344+ if buffer is None :
345+ self .replay (record )
346+ return
347+ buffer .append (record )
348+
349+ def replay (self , record : logging .LogRecord ):
350+ """Passes a record on to the handlers this one stands in for"""
351+ for handler in self .handlers :
352+ if record .levelno >= handler .level :
353+ handler .handle (record )
354+
355+
356+ def handler_chain (logger : logging .Logger ) -> List [logging .Handler ]:
357+ """The handlers a record logged on logger reaches, ancestors included
358+
359+ Mirrors what logging.Logger.callHandlers walks, so that replacing the
360+ handlers of one logger can stand in for the whole chain.
361+ """
362+ handlers = []
363+ current = logger
364+ while current :
365+ handlers .extend (current .handlers )
366+ if not current .propagate :
367+ break
368+ current = current .parent
369+ return handlers
370+
371+
312372def capture_output (
313- call : Callable , stdout : ThreadRoutedStdout , sink : List [str ], index : int
373+ call : Callable ,
374+ stdout : ThreadRoutedStdout ,
375+ log_handler : ThreadRoutedLogHandler ,
376+ printed : List [str ],
377+ logged : List [List [logging .LogRecord ]],
378+ index : int ,
314379) -> Callable :
315- """Wraps call so that what it prints ends up in sink[index] instead of stdout """
380+ """Wraps call so what it prints and logs ends up in the slot at index """
316381
317382 def wrapper ():
318383 buffer = stdout .register ()
384+ records = log_handler .register ()
319385 try :
320386 return call ()
321387 finally :
322388 stdout .unregister ()
323- sink [index ] = "" .join (buffer )
389+ log_handler .unregister ()
390+ printed [index ] = "" .join (buffer )
391+ logged [index ] = records
324392
325393 return wrapper
326394
@@ -361,7 +429,12 @@ def parallelize(
361429 # One slot per item, so the collected output can be replayed in input order
362430 # even though the tasks finish in whatever order they finish.
363431 outputs = ["" ] * len (items )
432+ records : List [List [logging .LogRecord ]] = [[] for _ in items ]
364433 stdout = ThreadRoutedStdout (sys .stdout )
434+ osw_logger = logging .getLogger ("osw" )
435+ log_handler = ThreadRoutedLogHandler (handler_chain (osw_logger ))
436+ saved_handlers = osw_logger .handlers
437+ saved_propagate = osw_logger .propagate
365438
366439 # run func concurrently with each item in iterable using asyncio
367440 async def _run_tasks ():
@@ -373,7 +446,12 @@ async def _run_tasks():
373446 loop .run_in_executor (
374447 None ,
375448 capture_output (
376- functools .partial (func , item , ** kwargs ), stdout , outputs , index
449+ functools .partial (func , item , ** kwargs ),
450+ stdout ,
451+ log_handler ,
452+ outputs ,
453+ records ,
454+ index ,
377455 ),
378456 )
379457 for index , item in enumerate (items )
@@ -396,6 +474,10 @@ async def _capture(task):
396474 return await gather (* tasks )
397475
398476 sys .stdout = stdout
477+ # standing in for the whole chain, so records from the osw.* child loggers
478+ # are routed too rather than propagating past this handler to the originals
479+ osw_logger .handlers = [log_handler ]
480+ osw_logger .propagate = False
399481 try :
400482 # Check if we are already in an event loop, e.g. in a Jupyter notebook
401483 try :
@@ -414,12 +496,16 @@ async def _capture(task):
414496 results = asyncio .run (_run_tasks ())
415497 finally :
416498 sys .stdout = stdout .stream
499+ osw_logger .handlers = saved_handlers
500+ osw_logger .propagate = saved_propagate
417501 if flush_at_end :
418502 # in the finally block, so a failed batch still reports what its
419503 # tasks had to say
420- for output in outputs :
504+ for output , task_records in zip ( outputs , records ) :
421505 if output :
422506 print (output , end = "" )
507+ for record in task_records :
508+ log_handler .replay (record )
423509
424510 return results
425511
0 commit comments