Skip to content

Commit 0c63fcf

Browse files
committed
add: multy Topic support
1 parent c926e20 commit 0c63fcf

5 files changed

Lines changed: 430 additions & 33 deletions

File tree

‎README.md‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,64 @@ broker.configure_producer(request_timeout_ms=100000)
3535
broker.configure_consumer(group_id="the best group ever.")
3636
```
3737

38+
## Multiple topics
39+
40+
By default `AioKafkaBroker` sends all tasks to `kafka_topic`.
41+
You can also configure the broker to listen to multiple topics and bind
42+
different tasks to different default topics.
43+
44+
```python
45+
from taskiq_aio_kafka import AioKafkaBroker
46+
from taskiq_aio_kafka.topic import Topic
47+
48+
broker = AioKafkaBroker(
49+
bootstrap_servers="localhost",
50+
kafka_topic="default-topic",
51+
kafka_topics=[
52+
Topic("emails"),
53+
Topic("reports"),
54+
],
55+
)
56+
57+
58+
@broker.task(topic="emails")
59+
async def send_email(user_id: int) -> None:
60+
print(f"Send email to {user_id}")
61+
62+
63+
@broker.task(topic="reports")
64+
async def build_report(report_id: int) -> None:
65+
print(f"Build report {report_id}")
66+
```
67+
68+
In this example the worker listens to `default-topic`, `emails`, and `reports`.
69+
When you call `send_email.kiq(...)`, the task is sent to `emails` by default.
70+
When you call `build_report.kiq(...)`, the task is sent to `reports` by default.
71+
72+
You can override a task topic for a single kick with `kicker().with_topic(...)`:
73+
74+
```python
75+
await send_email.kicker().with_topic("reports").kiq(user_id=1)
76+
```
77+
78+
Tasks without a custom topic keep the old behavior and are sent to `kafka_topic`.
79+
80+
```python
81+
@broker.task
82+
async def regular_task() -> None:
83+
print("This task goes to default-topic.")
84+
85+
86+
await regular_task.kiq()
87+
```
88+
3889
## Configuration
3990

4091
AioKafkaBroker parameters:
4192
* `bootstrap_servers` - url to kafka nodes. Can be either string or list of strings.
42-
* `kafka_topic` - custom topic in kafka.
93+
* `kafka_topic` - default topic in kafka.
94+
* `kafka_topics` - additional topics that worker should listen to.
4395
* `result_backend` - custom result backend.
4496
* `task_id_generator` - custom task_id genertaor.
4597
* `kafka_admin_client` - custom `kafka` admin client.
46-
* `delete_topic_on_shutdown` - flag to delete topic on broker shutdown.
98+
* `delete_topic_on_shutdown` - flag to delete topics on broker shutdown.

‎taskiq_aio_kafka/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""Taskiq integration with aiokafka."""
22

3-
from taskiq_aio_kafka.broker import AioKafkaBroker
3+
__all__ = ("AioKafkaBroker",)
44

5-
__all__ = ["AioKafkaBroker"]
5+
from taskiq_aio_kafka.broker import AioKafkaBroker

‎taskiq_aio_kafka/broker.py‎

Lines changed: 146 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,78 @@
1+
__all__ = ("AioKafkaBroker",)
2+
13
import asyncio
2-
from collections.abc import AsyncGenerator, Callable
4+
from collections.abc import AsyncGenerator, Callable, Iterable
35
from logging import getLogger
4-
from typing import Any, TypeVar
6+
from typing import Any, TypeAlias, TypeVar
57

68
from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
79
from kafka.admin import KafkaAdminClient, NewTopic
810
from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor
911
from kafka.partitioner.default import DefaultPartitioner
1012
from taskiq import AsyncResultBackend, BrokerMessage
1113
from taskiq.abc.broker import AsyncBroker
14+
from taskiq.decor import AsyncTaskiqDecoratedTask
15+
from taskiq.kicker import AsyncKicker
16+
from typing_extensions import ParamSpec
1217

1318
from taskiq_aio_kafka.exceptions import WrongAioKafkaBrokerParametersError
1419
from taskiq_aio_kafka.models import KafkaConsumerParameters, KafkaProducerParameters
20+
from taskiq_aio_kafka.topic import Topic
1521

1622
_T = TypeVar("_T")
23+
_FuncParams = ParamSpec("_FuncParams")
24+
_ReturnType = TypeVar("_ReturnType")
25+
TopicType: TypeAlias = str | NewTopic | Topic
26+
TASK_TOPIC_LABEL = "taskiq_aio_kafka_topic"
1727

1828

1929
logger = getLogger("taskiq.kafka_broker")
2030

2131

32+
def _get_topic_name(topic: TopicType) -> str:
33+
if isinstance(topic, str):
34+
return topic
35+
return topic.name
36+
37+
38+
class AioKafkaKicker(AsyncKicker[_FuncParams, _ReturnType]):
39+
"""Kicker that can override kafka topic for a task call."""
40+
41+
def with_topic(
42+
self,
43+
topic: TopicType,
44+
) -> "AioKafkaKicker[_FuncParams, _ReturnType]":
45+
"""Set kafka topic for current kick."""
46+
self.labels = {
47+
**self.labels,
48+
TASK_TOPIC_LABEL: _get_topic_name(topic),
49+
}
50+
return self
51+
52+
53+
class AioKafkaDecoratedTask(AsyncTaskiqDecoratedTask[_FuncParams, _ReturnType]):
54+
"""Taskiq decorated task with kafka-specific kicker."""
55+
56+
def kicker(self) -> AioKafkaKicker[_FuncParams, _ReturnType]:
57+
"""Return kafka-aware kicker."""
58+
return AioKafkaKicker(
59+
task_name=self.task_name,
60+
broker=self.broker,
61+
labels=self.labels,
62+
return_type=self.return_type,
63+
)
64+
65+
2266
class AioKafkaBroker(AsyncBroker):
2367
"""Broker that works with Kafka."""
2468

69+
task_topic_label = TASK_TOPIC_LABEL
70+
2571
def __init__( # noqa: PLR0913
2672
self,
2773
bootstrap_servers: str | list[str] | None,
28-
kafka_topic: NewTopic | None = None,
74+
kafka_topic: TopicType | None = None,
75+
kafka_topics: Iterable[TopicType] | None = None,
2976
result_backend: AsyncResultBackend[_T] | None = None,
3077
task_id_generator: Callable[[], str] | None = None,
3178
kafka_admin_client: KafkaAdminClient | None = None,
@@ -35,7 +82,8 @@ def __init__( # noqa: PLR0913
3582
"""Construct a new broker.
3683
3784
:param bootstrap_servers: string with url to kafka or list with urls.
38-
:param kafka_topic: kafka topic.
85+
:param kafka_topic: default kafka topic.
86+
:param kafka_topics: all kafka topics to listen.
3987
:param result_backend: custom result backend.
4088
:param task_id_generator: custom task_id generator.
4189
:param kafka_admin_client: configured KafkaAdminClient.
@@ -46,6 +94,7 @@ def __init__( # noqa: PLR0913
4694
aiokafka_consumer were specified but bootstrap_servers wasn't specified.
4795
"""
4896
super().__init__(result_backend, task_id_generator)
97+
self.decorator_class = AioKafkaDecoratedTask
4998

5099
if kafka_admin_client and not bootstrap_servers:
51100
raise WrongAioKafkaBrokerParametersError
@@ -54,11 +103,16 @@ def __init__( # noqa: PLR0913
54103

55104
self._loop: asyncio.AbstractEventLoop | None = loop
56105

57-
self._kafka_topic: NewTopic = kafka_topic or NewTopic(
58-
name="taskiq_topic",
59-
num_partitions=1,
60-
replication_factor=1,
61-
)
106+
self._kafka_topic: NewTopic = self._normalize_default_topic(kafka_topic)
107+
self._kafka_topics: dict[str, TopicType] = {
108+
self._kafka_topic.name: self._kafka_topic,
109+
}
110+
if kafka_topics is not None:
111+
for topic in kafka_topics:
112+
self._kafka_topics.setdefault(
113+
self._get_topic_name(topic),
114+
topic,
115+
)
62116

63117
self._aiokafka_producer_params: KafkaProducerParameters = (
64118
KafkaProducerParameters()
@@ -83,6 +137,48 @@ def __init__( # noqa: PLR0913
83137
self._is_producer_started = False
84138
self._is_consumer_started = False
85139

140+
@staticmethod
141+
def _get_topic_name(topic: TopicType) -> str:
142+
return _get_topic_name(topic)
143+
144+
@classmethod
145+
def _normalize_default_topic(
146+
cls,
147+
kafka_topic: TopicType | None,
148+
) -> NewTopic:
149+
if kafka_topic is None:
150+
return NewTopic(
151+
name="taskiq_topic",
152+
num_partitions=1,
153+
replication_factor=1,
154+
)
155+
if isinstance(kafka_topic, str):
156+
return NewTopic(
157+
name=kafka_topic,
158+
num_partitions=1,
159+
replication_factor=1,
160+
)
161+
if isinstance(kafka_topic, Topic):
162+
if kafka_topic.topic_config.declare:
163+
return kafka_topic.new_topic()
164+
return NewTopic(
165+
name=kafka_topic.name,
166+
num_partitions=1,
167+
replication_factor=1,
168+
)
169+
return kafka_topic
170+
171+
@classmethod
172+
def _get_declaration_topic(
173+
cls,
174+
topic: TopicType,
175+
) -> NewTopic | None:
176+
if isinstance(topic, NewTopic):
177+
return topic
178+
if isinstance(topic, Topic) and topic.topic_config.declare:
179+
return topic.new_topic()
180+
return None
181+
86182
def configure_producer(self, **producer_parameters: Any) -> None:
87183
"""Configure kafka producer.
88184
@@ -107,21 +203,41 @@ def configure_consumer(self, **consumer_parameters: Any) -> None:
107203
**consumer_parameters,
108204
)
109205

206+
def task( # type: ignore[override]
207+
self,
208+
task_name: str | Callable[..., Any] | None = None,
209+
*,
210+
topic: TopicType | None = None,
211+
**labels: Any,
212+
) -> Any:
213+
"""Decorate function and bind it to a kafka topic by default."""
214+
if topic is not None:
215+
topic_name = self._get_topic_name(topic)
216+
self._kafka_topics.setdefault(topic_name, topic)
217+
labels[self.task_topic_label] = topic_name
218+
219+
return super().task(
220+
task_name=task_name, # type: ignore[arg-type]
221+
**labels,
222+
)
223+
110224
async def startup(self) -> None:
111225
"""Setup AIOKafkaProducer, AIOKafkaConsumer and kafka topics.
112226
113-
We will have 2 topics for default and high priority.
114-
115227
Also we need to create AIOKafkaProducer and AIOKafkaConsumer
116228
if there are no producer and consumer passed.
117229
"""
118230
await super().startup()
119-
available_condition: bool = (
120-
self._kafka_topic.name not in self._kafka_admin_client.list_topics()
121-
)
122-
if available_condition:
231+
existed_topic_names = set(self._kafka_admin_client.list_topics())
232+
new_topics = [
233+
new_topic
234+
for topic in self._kafka_topics.values()
235+
if (new_topic := self._get_declaration_topic(topic)) is not None
236+
and new_topic.name not in existed_topic_names
237+
]
238+
if new_topics:
123239
self._kafka_admin_client.create_topics(
124-
new_topics=[self._kafka_topic],
240+
new_topics=new_topics,
125241
validate_only=False,
126242
)
127243

@@ -145,7 +261,7 @@ async def startup(self) -> None:
145261
partition_assignment_strategy
146262
)
147263
self._aiokafka_consumer = AIOKafkaConsumer(
148-
self._kafka_topic.name,
264+
*self._kafka_topics,
149265
bootstrap_servers=self._bootstrap_servers,
150266
loop=self._loop,
151267
**consumer_kwargs,
@@ -166,18 +282,16 @@ async def shutdown(self) -> None:
166282
if self._is_consumer_started:
167283
await self._aiokafka_consumer.stop()
168284

169-
topic_delete_condition: bool = all(
170-
(
171-
self._delete_topic_on_shutdown,
172-
self._kafka_topic.name in self._kafka_admin_client.list_topics(),
173-
),
174-
)
175-
176285
if self._kafka_admin_client:
177-
if topic_delete_condition:
178-
self._kafka_admin_client.delete_topics(
179-
[self._kafka_topic.name],
180-
)
286+
if self._delete_topic_on_shutdown:
287+
existed_topic_names = set(self._kafka_admin_client.list_topics())
288+
topic_names = [
289+
topic_name
290+
for topic_name in self._kafka_topics
291+
if topic_name in existed_topic_names
292+
]
293+
if topic_names:
294+
self._kafka_admin_client.delete_topics(topic_names)
181295
self._kafka_admin_client.close()
182296

183297
async def kick(self, message: BrokerMessage) -> None:
@@ -194,7 +308,10 @@ async def kick(self, message: BrokerMessage) -> None:
194308
if not self._is_producer_started:
195309
raise ValueError("Please run startup before kicking.")
196310

197-
topic_name: str = self._kafka_topic.name
311+
topic_name: str = message.labels.get(
312+
self.task_topic_label,
313+
self._kafka_topic.name,
314+
)
198315

199316
await self._aiokafka_producer.send_and_wait(
200317
topic=topic_name,

‎taskiq_aio_kafka/topic.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
__all__ = (
2+
"Topic",
3+
"TopicConfig",
4+
)
5+
6+
import dataclasses
7+
8+
from kafka.admin import NewTopic
9+
10+
11+
@dataclasses.dataclass
12+
class TopicConfig:
13+
"""Kafka topic declaration settings."""
14+
15+
declare: bool = False
16+
num_partitions: int = 1
17+
replication_factor: int = 1
18+
replica_assignments: dict[int, list[int]] = dataclasses.field(default_factory=dict)
19+
topic_configs: dict[str, str] = dataclasses.field(default_factory=dict)
20+
21+
22+
@dataclasses.dataclass
23+
class Topic:
24+
"""Taskiq kafka topic."""
25+
26+
name: str
27+
topic_config: TopicConfig = dataclasses.field(default_factory=TopicConfig)
28+
29+
def new_topic(self) -> NewTopic:
30+
"""Create kafka-python NewTopic instance."""
31+
if not self.topic_config.declare:
32+
raise ValueError("Topic declaration is disabled for this topic.")
33+
return NewTopic(
34+
name=self.name,
35+
num_partitions=self.topic_config.num_partitions,
36+
replication_factor=self.topic_config.replication_factor,
37+
replica_assignments=self.topic_config.replica_assignments,
38+
topic_configs=self.topic_config.topic_configs,
39+
)

0 commit comments

Comments
 (0)