|
| 1 | +import contextlib |
| 2 | +import dataclasses |
| 3 | +import datetime as dt |
| 4 | +import logging |
| 5 | +from collections.abc import AsyncGenerator |
| 6 | + |
| 7 | +from sqlalchemy import Table, delete, func, select, tuple_ |
| 8 | +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine |
| 9 | + |
| 10 | +from sqlalchemy_inbox.schema import validate_table_identifiers |
| 11 | + |
| 12 | + |
| 13 | +_LOGGER = logging.getLogger(__name__) |
| 14 | + |
| 15 | + |
| 16 | +@dataclasses.dataclass(kw_only=True, frozen=True, slots=True) |
| 17 | +class CleanupResult: |
| 18 | + cutoff: dt.datetime |
| 19 | + deleted_count: int |
| 20 | + |
| 21 | + |
| 22 | +class InboxCleaner: |
| 23 | + """Deletes markers older than *retention*, one transaction per batch. |
| 24 | +
|
| 25 | + Opens its own connections from *engine* but never disposes it: the caller owns the engine. |
| 26 | + """ |
| 27 | + |
| 28 | + def __init__(self, *, engine: AsyncEngine, table: Table, retention: dt.timedelta, batch_size: int = 1000) -> None: |
| 29 | + if not isinstance(engine, AsyncEngine): |
| 30 | + msg = "InboxCleaner requires an sqlalchemy.ext.asyncio.AsyncEngine" |
| 31 | + raise TypeError(msg) |
| 32 | + validate_table_identifiers(table.name) |
| 33 | + if retention <= dt.timedelta(0): |
| 34 | + msg = f"retention must be positive (got {retention})" |
| 35 | + raise ValueError(msg) |
| 36 | + if batch_size < 1: |
| 37 | + msg = f"batch_size must be positive (got {batch_size})" |
| 38 | + raise ValueError(msg) |
| 39 | + self._engine = engine |
| 40 | + self._table = table |
| 41 | + self._retention = retention |
| 42 | + self._batch_size = batch_size |
| 43 | + |
| 44 | + async def cleanup(self) -> CleanupResult: |
| 45 | + """Take one cutoff from the database clock, then delete the markers older than it until a batch deletes nothing. |
| 46 | +
|
| 47 | + Each batch commits on its own and skips markers other transactions hold. If a batch fails or the task |
| 48 | + is cancelled, that batch rolls back, earlier batches stay deleted, and the error propagates. |
| 49 | + """ |
| 50 | + async with self._transaction() as connection: |
| 51 | + cutoff = (await connection.execute(select(func.now() - self._retention))).scalar_one() |
| 52 | + deleted_count = 0 |
| 53 | + while deleted := await self._delete_batch(cutoff): |
| 54 | + deleted_count += deleted |
| 55 | + _LOGGER.debug("Inbox cleanup batch deleted %d markers (%d so far)", deleted, deleted_count) |
| 56 | + _LOGGER.info("Inbox cleanup deleted %d markers older than %s", deleted_count, cutoff.isoformat()) |
| 57 | + return CleanupResult(cutoff=cutoff, deleted_count=deleted_count) |
| 58 | + |
| 59 | + async def _delete_batch(self, cutoff: dt.datetime) -> int: |
| 60 | + key = tuple_(self._table.c.scope, self._table.c.message_id) |
| 61 | + expired = ( |
| 62 | + select(self._table.c.scope, self._table.c.message_id) |
| 63 | + .where(self._table.c.processed_at < cutoff) |
| 64 | + .limit(self._batch_size) |
| 65 | + .with_for_update(skip_locked=True) |
| 66 | + ) |
| 67 | + async with self._transaction() as connection: |
| 68 | + result = await connection.execute( |
| 69 | + delete(self._table).where(key.in_(expired)).returning(self._table.c.scope) |
| 70 | + ) |
| 71 | + return len(result.all()) |
| 72 | + |
| 73 | + @contextlib.asynccontextmanager |
| 74 | + async def _transaction(self) -> AsyncGenerator[AsyncConnection]: |
| 75 | + async with self._engine.connect() as connection: |
| 76 | + await connection.execution_options(isolation_level="READ COMMITTED") |
| 77 | + async with connection.begin(): |
| 78 | + yield connection |
0 commit comments