Skip to content

Commit 2430204

Browse files
committed
feat(logging): set up the osw logger and route it through parallelize
- osw logs at INFO by default, with a one-time notice on import - set_log_level, disable_logging, enable_logging and OSW_LOG_LEVEL - ThreadRoutedLogHandler, the log counterpart of ThreadRoutedStdout - parallelize buffers worker records and replays them in input order
1 parent 1dc554f commit 2430204

2 files changed

Lines changed: 205 additions & 5 deletions

File tree

‎src/osw/__init__.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import logging
2+
import os
3+
import sys
14
from importlib.metadata import PackageNotFoundError, version # pragma: no cover
25

36
try:
@@ -8,3 +11,114 @@
811
__version__ = "unknown"
912
finally:
1013
del version, PackageNotFoundError
14+
15+
LOG_LEVEL_ENV_VAR = "OSW_LOG_LEVEL"
16+
"""Name of the environment variable read for the initial log level. Accepts a
17+
level name, a level number, or 'OFF' to start silent."""
18+
DEFAULT_LOG_LEVEL = logging.INFO
19+
"""Level the osw logger starts at when the environment variable is unset"""
20+
LOG_FORMAT = "[%(levelname)s] %(name)s: %(message)s"
21+
OFF = logging.CRITICAL + 1
22+
"""Level above every real record, so setting it silences the logger"""
23+
24+
_DEFAULT_HANDLER_NAME = "osw-default"
25+
_logger = logging.getLogger(__name__)
26+
27+
28+
def _parse_level(value: str) -> int:
29+
"""Turns the environment variable into a level, tolerating junk
30+
31+
Parameters
32+
----------
33+
value
34+
a level name such as 'DEBUG', a level number such as '10', or 'OFF'
35+
36+
Returns
37+
-------
38+
the level, or DEFAULT_LOG_LEVEL if the value names none
39+
"""
40+
cleaned = value.strip().upper()
41+
if cleaned in ("OFF", "NONE", "SILENT"):
42+
return OFF
43+
if cleaned.isdigit():
44+
return int(cleaned)
45+
# getLevelName maps an unknown name to the string 'Level %s', not to a number
46+
level = logging.getLevelName(cleaned)
47+
if isinstance(level, int):
48+
return level
49+
return DEFAULT_LOG_LEVEL
50+
51+
52+
def enable_logging(level=None, stream=None) -> logging.Handler:
53+
"""Attaches the default handler to the osw logger
54+
55+
Called once on import, so osw reports what it is doing without the calling
56+
code configuring anything. Call it again to restore the handler after
57+
disable_logging(), or to send the records to another stream.
58+
59+
Parameters
60+
----------
61+
level
62+
a level name or number. Defaults to the environment variable, then to
63+
DEFAULT_LOG_LEVEL.
64+
stream
65+
where to write. Defaults to sys.stdout, which is where the print calls
66+
this replaced used to go.
67+
68+
Returns
69+
-------
70+
the attached handler
71+
"""
72+
if level is None:
73+
level = _parse_level(os.environ.get(LOG_LEVEL_ENV_VAR, ""))
74+
disable_logging()
75+
handler = logging.StreamHandler(sys.stdout if stream is None else stream)
76+
handler.setFormatter(logging.Formatter(LOG_FORMAT))
77+
handler.set_name(_DEFAULT_HANDLER_NAME)
78+
_logger.addHandler(handler)
79+
_logger.setLevel(level)
80+
# the records are handled here, so passing them to the root logger as well
81+
# would print everything twice for anyone who called basicConfig()
82+
_logger.propagate = False
83+
return handler
84+
85+
86+
def disable_logging():
87+
"""Removes the default handler, leaving osw silent
88+
89+
Only the handler osw attached itself is removed, so a handler the calling
90+
code added to the 'osw' logger keeps receiving records. Records propagate
91+
to the root logger again, which is what a library is normally expected to
92+
do, and lets the calling code take over with basicConfig().
93+
"""
94+
for handler in list(_logger.handlers):
95+
if handler.get_name() == _DEFAULT_HANDLER_NAME:
96+
_logger.removeHandler(handler)
97+
handler.close()
98+
_logger.propagate = True
99+
100+
101+
def set_log_level(level):
102+
"""Sets how much osw reports
103+
104+
Parameters
105+
----------
106+
level
107+
a level name or number, e.g. 'DEBUG', 'WARNING', or logging.WARNING.
108+
Pass osw.OFF to silence osw without detaching the handler.
109+
"""
110+
_logger.setLevel(level)
111+
112+
113+
enable_logging()
114+
115+
if os.environ.get(LOG_LEVEL_ENV_VAR) is None:
116+
# once per interpreter, since this module is only imported once. Sent
117+
# through the logger itself, so it disappears with everything else as soon
118+
# as the level is raised or the environment variable is set.
119+
_logger.info(
120+
"osw logs at %s. Use osw.set_log_level('WARNING') to see less, "
121+
"osw.disable_logging() to switch it off, or set %s=OFF before import.",
122+
logging.getLevelName(_logger.level),
123+
LOG_LEVEL_ENV_VAR,
124+
)

‎src/osw/utils/util.py‎

Lines changed: 91 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import asyncio
22
import functools
3+
import logging
34
import sys
45
import threading
56
from asyncio import Queue
@@ -309,18 +310,85 @@ def __getattr__(self, name):
309310
return getattr(self.stream, name)
310311

311312

313+
class ThreadRoutedLogHandler(logging.Handler):
314+
"""A log handler that routes each worker thread's records to its own buffer
315+
316+
The counterpart of ThreadRoutedStdout for the modules that log rather than
317+
print. Installed on the 'osw' logger for the duration of a parallelize()
318+
call, in place of the handlers a record would otherwise reach. Threads that
319+
have not registered a buffer, the calling thread among them, are forwarded
320+
to those handlers unchanged.
321+
322+
Routing has to happen at the handler, not at a filter on the 'osw' logger. A
323+
logger's filters only run for records logged on that logger itself, so they
324+
would never see the records from osw.wtsite and the other child loggers.
325+
"""
326+
327+
def __init__(self, handlers: List[logging.Handler]):
328+
super().__init__()
329+
self.handlers = handlers
330+
self._local = threading.local()
331+
332+
def register(self) -> List[logging.LogRecord]:
333+
"""Gives the calling thread its own buffer and returns it"""
334+
buffer = []
335+
self._local.buffer = buffer
336+
return buffer
337+
338+
def unregister(self):
339+
"""Sends the calling thread's records back to the wrapped handlers"""
340+
self._local.buffer = None
341+
342+
def emit(self, record: logging.LogRecord):
343+
buffer = getattr(self._local, "buffer", None)
344+
if buffer is None:
345+
self.replay(record)
346+
return
347+
buffer.append(record)
348+
349+
def replay(self, record: logging.LogRecord):
350+
"""Passes a record on to the handlers this one stands in for"""
351+
for handler in self.handlers:
352+
if record.levelno >= handler.level:
353+
handler.handle(record)
354+
355+
356+
def handler_chain(logger: logging.Logger) -> List[logging.Handler]:
357+
"""The handlers a record logged on logger reaches, ancestors included
358+
359+
Mirrors what logging.Logger.callHandlers walks, so that replacing the
360+
handlers of one logger can stand in for the whole chain.
361+
"""
362+
handlers = []
363+
current = logger
364+
while current:
365+
handlers.extend(current.handlers)
366+
if not current.propagate:
367+
break
368+
current = current.parent
369+
return handlers
370+
371+
312372
def capture_output(
313-
call: Callable, stdout: ThreadRoutedStdout, sink: List[str], index: int
373+
call: Callable,
374+
stdout: ThreadRoutedStdout,
375+
log_handler: ThreadRoutedLogHandler,
376+
printed: List[str],
377+
logged: List[List[logging.LogRecord]],
378+
index: int,
314379
) -> Callable:
315-
"""Wraps call so that what it prints ends up in sink[index] instead of stdout"""
380+
"""Wraps call so what it prints and logs ends up in the slot at index"""
316381

317382
def wrapper():
318383
buffer = stdout.register()
384+
records = log_handler.register()
319385
try:
320386
return call()
321387
finally:
322388
stdout.unregister()
323-
sink[index] = "".join(buffer)
389+
log_handler.unregister()
390+
printed[index] = "".join(buffer)
391+
logged[index] = records
324392

325393
return wrapper
326394

@@ -361,7 +429,12 @@ def parallelize(
361429
# One slot per item, so the collected output can be replayed in input order
362430
# even though the tasks finish in whatever order they finish.
363431
outputs = [""] * len(items)
432+
records: List[List[logging.LogRecord]] = [[] for _ in items]
364433
stdout = ThreadRoutedStdout(sys.stdout)
434+
osw_logger = logging.getLogger("osw")
435+
log_handler = ThreadRoutedLogHandler(handler_chain(osw_logger))
436+
saved_handlers = osw_logger.handlers
437+
saved_propagate = osw_logger.propagate
365438

366439
# run func concurrently with each item in iterable using asyncio
367440
async def _run_tasks():
@@ -373,7 +446,12 @@ async def _run_tasks():
373446
loop.run_in_executor(
374447
None,
375448
capture_output(
376-
functools.partial(func, item, **kwargs), stdout, outputs, index
449+
functools.partial(func, item, **kwargs),
450+
stdout,
451+
log_handler,
452+
outputs,
453+
records,
454+
index,
377455
),
378456
)
379457
for index, item in enumerate(items)
@@ -396,6 +474,10 @@ async def _capture(task):
396474
return await gather(*tasks)
397475

398476
sys.stdout = stdout
477+
# standing in for the whole chain, so records from the osw.* child loggers
478+
# are routed too rather than propagating past this handler to the originals
479+
osw_logger.handlers = [log_handler]
480+
osw_logger.propagate = False
399481
try:
400482
# Check if we are already in an event loop, e.g. in a Jupyter notebook
401483
try:
@@ -414,12 +496,16 @@ async def _capture(task):
414496
results = asyncio.run(_run_tasks())
415497
finally:
416498
sys.stdout = stdout.stream
499+
osw_logger.handlers = saved_handlers
500+
osw_logger.propagate = saved_propagate
417501
if flush_at_end:
418502
# in the finally block, so a failed batch still reports what its
419503
# tasks had to say
420-
for output in outputs:
504+
for output, task_records in zip(outputs, records):
421505
if output:
422506
print(output, end="")
507+
for record in task_records:
508+
log_handler.replay(record)
423509

424510
return results
425511

0 commit comments

Comments
 (0)