feat: add support for log extras

Author: ErikKalkoken

.gitignore

@@ -205,3 +205,6 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+
+# VS Code
+.vscode/

Makefile

@@ -1,2 +1,3 @@
-test:
-	python -m unittest discover
+coverage:
+	coverage run -m unittest discover tests/
+	coverage report -m

README.md

@@ -8,4 +8,15 @@ A Python log handler for Victoria Logs.
 [![codecov](https://codecov.io/gh/ErikKalkoken/python-victorialogs-handler/graph/badge.svg?token=2pPb3lid2k)](https://codecov.io/gh/ErikKalkoken/python-victorialogs-handler)
 [![license](https://img.shields.io/badge/license-MIT-green)](https://gitlab.com/ErikKalkoken/python-victorialogs-handler/-/blob/master/LICENSE)
 
-In development...
+STATUS: In development
+
+## Description
+
+This package provides a Python log handler for Victoria Logs. The log handler is designed to work with Python's default logging module and will send all log events to a configured Victoria Logs server for log ingestion.
+
+## Key Features
+
+- Asynchronous design: Log events are queued and then processed in a separate thread so that the performance impact on the main program remains minimal.
+- Request batching: Log events are processed without delay. Multiple log events will be sent in a single request as batch to minimize the number of requests to the VictoriaLogs server.
+- Supports extras: Extra fields are supported. This includes non-standard types like sets and datetime objects. Fields that can not be serialized to JSON (e.g. functions) will be converted into their string representation
+- Supports dict config: The handler supports dict configuration, e.g. for a Django server

src/vlogs_handler/__init__.py

@@ -3,10 +3,7 @@
 from .handler import VictoriaLogsHandler  # noqa: F401
 
 __title__ = "Victoria Logs Handler"
-__version__ = "0.1.0dev1"
+__version__ = "0.1.0dev2"
 
 # TO-DOs
-# [x] Batch requests
-# [x] tests
-# [x] timeout configurable
 # [ ] Options for defining stream

src/vlogs_handler/handler.py

@@ -1,5 +1,6 @@
 """Module handler provides the implementation of the vlogs handler."""
 
+import datetime as dt
 import io
 import json
 import logging
@@ -16,6 +17,31 @@
 _VLOGS_PARAMS = "_stream_fields=stream&_time_field=timestamp&_msg_field=message"
 
 
+_STANDARD_ATTRS = {
+    "args",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "message",
+    "module",
+    "msecs",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+}
+
+
 class VictoriaLogsHandler(logging.Handler):
     """VictoriaLogsHandler dispatches log events to a Victoria Logs server.
 
@@ -73,6 +99,11 @@ def format_log_entry(self, record: logging.LogRecord) -> Dict[str, Any]:
         }
         if record.exc_info:
             entry["exception"] = _format_exception(record.exc_info)
+
+        for k, v in record.__dict__.items():
+            if k not in _STANDARD_ATTRS:
+                entry[k] = v
+
         return entry
 
     def _worker(self):
@@ -92,7 +123,8 @@ def _send(self, entries: List[Dict[str, Any]]):
         lines = []
         for entry in entries:
             try:
-                lines.append(json.dumps(entry))
+                data = json.dumps(entry, cls=JSONEncoderPlus)
+                lines.append(data)
             except Exception as ex:
                 log.exception("convert entry to JSON", ex, entry=entry)
                 continue
@@ -139,3 +171,24 @@ def _calc_stream_from_record(record: logging.LogRecord):
         else:
             stream = record.name
     return stream
+
+
+class JSONEncoderPlus(json.JSONEncoder):
+    """JSONEncoderPlus is an improved encoder that can convert dates and does not break.
+
+    Instead of breaking it will return a string representation
+    for unserializable fields.
+    """
+
+    def default(self, obj):
+        if isinstance(obj, (dt.datetime, dt.date)):
+            return obj.isoformat()
+
+        if isinstance(obj, set):
+            return list(obj)
+
+        try:
+            return super().default(obj)
+        except TypeError:
+            # If we can't serialize it, return None or a string representation
+            return str(obj)

tests/test_handler.py

@@ -1,11 +1,13 @@
+import datetime as dt
 import json
 import logging
-import time
 import unittest
+from threading import Event
 
 import requests_mock
 
 from vlogs_handler import VictoriaLogsHandler
+from vlogs_handler.handler import JSONEncoderPlus
 
 
 @requests_mock.Mocker()
@@ -15,29 +17,87 @@ def setUp(self):
         self.logger = logging.getLogger("test_logger")
         self.logger.addHandler(self.handler)
         self.logger.setLevel(logging.INFO)
+        self.ready_event = Event()
 
     def tearDown(self):
         self.logger.removeHandler(self.handler)
 
-    def test_handler_sends_log(self, m: requests_mock.Mocker):
+    def text_callback(self, request, context):
+        self.ready_event.set()
+        return ""
+
+    def test_handler_should_send_normal_log(self, m: requests_mock.Mocker):
         # given
         m.register_uri(
             "POST",
             "http://localhost:30123/insert/jsonline",
             status_code=200,
+            text=self.text_callback,
         )
 
         # when
         self.logger.info("Alpha")
-        time.sleep(0.5)
 
         # then
+        called_in_time = self.ready_event.wait(timeout=2.0)
+        self.assertTrue(called_in_time)
+
+        self.assertEqual(m.call_count, 1)
+        got = m.last_request.json()  # type: ignore
+        self.assertEqual(got["stream"], "test_logger")
+        self.assertEqual(got["level"], "INFO")
+        self.assertEqual(got["logger"], "test_logger")
+        self.assertEqual(got["message"], "Alpha")
+
+    def test_handler_should_send_log_with_extras(self, m: requests_mock.Mocker):
+        # given
+        m.register_uri(
+            "POST",
+            "http://localhost:30123/insert/jsonline",
+            status_code=200,
+            text=self.text_callback,
+        )
+        my_date = dt.datetime(2026, 1, 11, 12, 15, 42, 99, tzinfo=dt.UTC)
+
+        # when
+        self.logger.info("Alpha", extra={"planet": "Jupiter", "deadline": my_date})
+
+        # then
+        called_in_time = self.ready_event.wait(timeout=2.0)
+        self.assertTrue(called_in_time)
+
+        self.assertEqual(m.call_count, 1)
+        got = m.last_request.json()  # type: ignore
+        self.assertEqual(got["message"], "Alpha")
+        self.assertEqual(got["planet"], "Jupiter")
+        self.assertEqual(got["deadline"], "2026-01-11T12:15:42.000099+00:00")
+
+    def test_handler_should_send_exception_log(self, m: requests_mock.Mocker):
+        # given
+        m.register_uri(
+            "POST",
+            "http://localhost:30123/insert/jsonline",
+            status_code=200,
+            text=self.text_callback,
+        )
+
+        # when
+        try:
+            _ = 1 / 0
+        except ZeroDivisionError:
+            self.logger.exception("Bravo")
+
+        # then
+        called_in_time = self.ready_event.wait(timeout=2.0)
+        self.assertTrue(called_in_time)
+
         self.assertEqual(m.call_count, 1)
-        data = m.last_request.json()  # type: ignore
-        self.assertEqual(data["stream"], "test_logger")
-        self.assertEqual(data["level"], "INFO")
-        self.assertEqual(data["logger"], "test_logger")
-        self.assertEqual(data["message"], "Alpha")
+        got = m.last_request.json()  # type: ignore
+        self.assertEqual(got["stream"], "test_logger")
+        self.assertEqual(got["level"], "ERROR")
+        self.assertEqual(got["logger"], "test_logger")
+        self.assertEqual(got["message"], "Bravo")
+        self.assertIn("ZeroDivisionError", got["exception"])
 
 
 @requests_mock.Mocker()
@@ -49,25 +109,33 @@ def setUp(self):
         self.logger = logging.getLogger("test_logger")
         self.logger.addHandler(self.handler)
         self.logger.setLevel(logging.INFO)
+        self.ready_event = Event()
 
     def tearDown(self):
         self.logger.removeHandler(self.handler)
 
+    def text_callback(self, request, context):
+        self.ready_event.set()
+        return ""
+
     def test_handler_sends_log(self, m: requests_mock.Mocker):
         # given
         m.register_uri(
             "POST",
             "http://localhost:30123/insert/jsonline",
             status_code=200,
+            text=self.text_callback,
         )
 
         # when
         self.logger.info("Alpha")
         self.logger.info("Bravo")
         self.handler.start()
-        time.sleep(0.5)
 
         # then
+        called_in_time = self.ready_event.wait(timeout=2.0)
+        self.assertTrue(called_in_time)
+
         self.assertEqual(m.call_count, 1)
         data = m.last_request.text  # type: ignore
         lines = data.splitlines()
@@ -78,3 +146,38 @@ def test_handler_sends_log(self, m: requests_mock.Mocker):
 
         e2 = json.loads(lines[1])
         self.assertEqual(e2["message"], "Bravo")
+        self.assertEqual(e2["message"], "Bravo")
+
+
+class TestJSONEncoderPlus(unittest.TestCase):
+    def test_should_encode(self):
+        # given
+        def my_func():
+            pass
+
+        my_date = dt.datetime(2026, 1, 11, 12, 15, 42, 99, tzinfo=dt.UTC)
+        data = {
+            "class": JSONEncoderPlus,
+            "date": my_date.date(),
+            "datetime": my_date,
+            "float": 1.23,
+            "func": my_func,
+            "integer": 1,
+            "set": {1, 2, 3},
+            "text": "Alpha",
+        }
+
+        # when
+        got = json.loads(json.dumps(data, cls=JSONEncoderPlus))
+
+        # then
+        self.assertEqual(
+            got["class"], "<class 'vlogs_handler.handler.JSONEncoderPlus'>"
+        )
+        self.assertEqual(got["date"], "2026-01-11")
+        self.assertEqual(got["datetime"], "2026-01-11T12:15:42.000099+00:00")
+        self.assertEqual(got["float"], 1.23)
+        self.assertIn("my_func at", got["func"])
+        self.assertEqual(got["integer"], 1)
+        self.assertEqual(got["set"], [1, 2, 3])
+        self.assertEqual(got["text"], "Alpha")