1+ __all__ = ("AioKafkaBroker" ,)
2+
13import asyncio
2- from collections .abc import AsyncGenerator , Callable
4+ from collections .abc import AsyncGenerator , Callable , Iterable
35from logging import getLogger
4- from typing import Any , TypeVar
6+ from typing import Any , TypeAlias , TypeVar
57
68from aiokafka import AIOKafkaConsumer , AIOKafkaProducer
79from kafka .admin import KafkaAdminClient , NewTopic
810from kafka .coordinator .assignors .roundrobin import RoundRobinPartitionAssignor
911from kafka .partitioner .default import DefaultPartitioner
1012from taskiq import AsyncResultBackend , BrokerMessage
1113from taskiq .abc .broker import AsyncBroker
14+ from taskiq .decor import AsyncTaskiqDecoratedTask
15+ from taskiq .kicker import AsyncKicker
16+ from typing_extensions import ParamSpec
1217
1318from taskiq_aio_kafka .exceptions import WrongAioKafkaBrokerParametersError
1419from taskiq_aio_kafka .models import KafkaConsumerParameters , KafkaProducerParameters
20+ from taskiq_aio_kafka .topic import Topic
1521
1622_T = TypeVar ("_T" )
23+ _FuncParams = ParamSpec ("_FuncParams" )
24+ _ReturnType = TypeVar ("_ReturnType" )
25+ TopicType : TypeAlias = str | NewTopic | Topic
26+ TASK_TOPIC_LABEL = "taskiq_aio_kafka_topic"
1727
1828
1929logger = getLogger ("taskiq.kafka_broker" )
2030
2131
32+ def _get_topic_name (topic : TopicType ) -> str :
33+ if isinstance (topic , str ):
34+ return topic
35+ return topic .name
36+
37+
38+ class AioKafkaKicker (AsyncKicker [_FuncParams , _ReturnType ]):
39+ """Kicker that can override kafka topic for a task call."""
40+
41+ def with_topic (
42+ self ,
43+ topic : TopicType ,
44+ ) -> "AioKafkaKicker[_FuncParams, _ReturnType]" :
45+ """Set kafka topic for current kick."""
46+ self .labels = {
47+ ** self .labels ,
48+ TASK_TOPIC_LABEL : _get_topic_name (topic ),
49+ }
50+ return self
51+
52+
53+ class AioKafkaDecoratedTask (AsyncTaskiqDecoratedTask [_FuncParams , _ReturnType ]):
54+ """Taskiq decorated task with kafka-specific kicker."""
55+
56+ def kicker (self ) -> AioKafkaKicker [_FuncParams , _ReturnType ]:
57+ """Return kafka-aware kicker."""
58+ return AioKafkaKicker (
59+ task_name = self .task_name ,
60+ broker = self .broker ,
61+ labels = self .labels ,
62+ return_type = self .return_type ,
63+ )
64+
65+
2266class AioKafkaBroker (AsyncBroker ):
2367 """Broker that works with Kafka."""
2468
69+ task_topic_label = TASK_TOPIC_LABEL
70+
2571 def __init__ ( # noqa: PLR0913
2672 self ,
2773 bootstrap_servers : str | list [str ] | None ,
28- kafka_topic : NewTopic | None = None ,
74+ kafka_topic : TopicType | None = None ,
75+ kafka_topics : Iterable [TopicType ] | None = None ,
2976 result_backend : AsyncResultBackend [_T ] | None = None ,
3077 task_id_generator : Callable [[], str ] | None = None ,
3178 kafka_admin_client : KafkaAdminClient | None = None ,
@@ -35,7 +82,8 @@ def __init__( # noqa: PLR0913
3582 """Construct a new broker.
3683
3784 :param bootstrap_servers: string with url to kafka or list with urls.
38- :param kafka_topic: kafka topic.
85+ :param kafka_topic: default kafka topic.
86+ :param kafka_topics: all kafka topics to listen.
3987 :param result_backend: custom result backend.
4088 :param task_id_generator: custom task_id generator.
4189 :param kafka_admin_client: configured KafkaAdminClient.
@@ -46,6 +94,7 @@ def __init__( # noqa: PLR0913
4694 aiokafka_consumer were specified but bootstrap_servers wasn't specified.
4795 """
4896 super ().__init__ (result_backend , task_id_generator )
97+ self .decorator_class = AioKafkaDecoratedTask
4998
5099 if kafka_admin_client and not bootstrap_servers :
51100 raise WrongAioKafkaBrokerParametersError
@@ -54,11 +103,16 @@ def __init__( # noqa: PLR0913
54103
55104 self ._loop : asyncio .AbstractEventLoop | None = loop
56105
57- self ._kafka_topic : NewTopic = kafka_topic or NewTopic (
58- name = "taskiq_topic" ,
59- num_partitions = 1 ,
60- replication_factor = 1 ,
61- )
106+ self ._kafka_topic : NewTopic = self ._normalize_default_topic (kafka_topic )
107+ self ._kafka_topics : dict [str , TopicType ] = {
108+ self ._kafka_topic .name : self ._kafka_topic ,
109+ }
110+ if kafka_topics is not None :
111+ for topic in kafka_topics :
112+ self ._kafka_topics .setdefault (
113+ self ._get_topic_name (topic ),
114+ topic ,
115+ )
62116
63117 self ._aiokafka_producer_params : KafkaProducerParameters = (
64118 KafkaProducerParameters ()
@@ -83,6 +137,48 @@ def __init__( # noqa: PLR0913
83137 self ._is_producer_started = False
84138 self ._is_consumer_started = False
85139
140+ @staticmethod
141+ def _get_topic_name (topic : TopicType ) -> str :
142+ return _get_topic_name (topic )
143+
144+ @classmethod
145+ def _normalize_default_topic (
146+ cls ,
147+ kafka_topic : TopicType | None ,
148+ ) -> NewTopic :
149+ if kafka_topic is None :
150+ return NewTopic (
151+ name = "taskiq_topic" ,
152+ num_partitions = 1 ,
153+ replication_factor = 1 ,
154+ )
155+ if isinstance (kafka_topic , str ):
156+ return NewTopic (
157+ name = kafka_topic ,
158+ num_partitions = 1 ,
159+ replication_factor = 1 ,
160+ )
161+ if isinstance (kafka_topic , Topic ):
162+ if kafka_topic .topic_config .declare :
163+ return kafka_topic .new_topic ()
164+ return NewTopic (
165+ name = kafka_topic .name ,
166+ num_partitions = 1 ,
167+ replication_factor = 1 ,
168+ )
169+ return kafka_topic
170+
171+ @classmethod
172+ def _get_declaration_topic (
173+ cls ,
174+ topic : TopicType ,
175+ ) -> NewTopic | None :
176+ if isinstance (topic , NewTopic ):
177+ return topic
178+ if isinstance (topic , Topic ) and topic .topic_config .declare :
179+ return topic .new_topic ()
180+ return None
181+
86182 def configure_producer (self , ** producer_parameters : Any ) -> None :
87183 """Configure kafka producer.
88184
@@ -107,21 +203,41 @@ def configure_consumer(self, **consumer_parameters: Any) -> None:
107203 ** consumer_parameters ,
108204 )
109205
206+ def task ( # type: ignore[override]
207+ self ,
208+ task_name : str | Callable [..., Any ] | None = None ,
209+ * ,
210+ topic : TopicType | None = None ,
211+ ** labels : Any ,
212+ ) -> Any :
213+ """Decorate function and bind it to a kafka topic by default."""
214+ if topic is not None :
215+ topic_name = self ._get_topic_name (topic )
216+ self ._kafka_topics .setdefault (topic_name , topic )
217+ labels [self .task_topic_label ] = topic_name
218+
219+ return super ().task (
220+ task_name = task_name , # type: ignore[arg-type]
221+ ** labels ,
222+ )
223+
110224 async def startup (self ) -> None :
111225 """Setup AIOKafkaProducer, AIOKafkaConsumer and kafka topics.
112226
113- We will have 2 topics for default and high priority.
114-
115227 Also we need to create AIOKafkaProducer and AIOKafkaConsumer
116228 if there are no producer and consumer passed.
117229 """
118230 await super ().startup ()
119- available_condition : bool = (
120- self ._kafka_topic .name not in self ._kafka_admin_client .list_topics ()
121- )
122- if available_condition :
231+ existed_topic_names = set (self ._kafka_admin_client .list_topics ())
232+ new_topics = [
233+ new_topic
234+ for topic in self ._kafka_topics .values ()
235+ if (new_topic := self ._get_declaration_topic (topic )) is not None
236+ and new_topic .name not in existed_topic_names
237+ ]
238+ if new_topics :
123239 self ._kafka_admin_client .create_topics (
124- new_topics = [ self . _kafka_topic ] ,
240+ new_topics = new_topics ,
125241 validate_only = False ,
126242 )
127243
@@ -145,7 +261,7 @@ async def startup(self) -> None:
145261 partition_assignment_strategy
146262 )
147263 self ._aiokafka_consumer = AIOKafkaConsumer (
148- self ._kafka_topic . name ,
264+ * self ._kafka_topics ,
149265 bootstrap_servers = self ._bootstrap_servers ,
150266 loop = self ._loop ,
151267 ** consumer_kwargs ,
@@ -166,18 +282,16 @@ async def shutdown(self) -> None:
166282 if self ._is_consumer_started :
167283 await self ._aiokafka_consumer .stop ()
168284
169- topic_delete_condition : bool = all (
170- (
171- self ._delete_topic_on_shutdown ,
172- self ._kafka_topic .name in self ._kafka_admin_client .list_topics (),
173- ),
174- )
175-
176285 if self ._kafka_admin_client :
177- if topic_delete_condition :
178- self ._kafka_admin_client .delete_topics (
179- [self ._kafka_topic .name ],
180- )
286+ if self ._delete_topic_on_shutdown :
287+ existed_topic_names = set (self ._kafka_admin_client .list_topics ())
288+ topic_names = [
289+ topic_name
290+ for topic_name in self ._kafka_topics
291+ if topic_name in existed_topic_names
292+ ]
293+ if topic_names :
294+ self ._kafka_admin_client .delete_topics (topic_names )
181295 self ._kafka_admin_client .close ()
182296
183297 async def kick (self , message : BrokerMessage ) -> None :
@@ -194,7 +308,10 @@ async def kick(self, message: BrokerMessage) -> None:
194308 if not self ._is_producer_started :
195309 raise ValueError ("Please run startup before kicking." )
196310
197- topic_name : str = self ._kafka_topic .name
311+ topic_name : str = message .labels .get (
312+ self .task_topic_label ,
313+ self ._kafka_topic .name ,
314+ )
198315
199316 await self ._aiokafka_producer .send_and_wait (
200317 topic = topic_name ,
0 commit comments