Skip to content

Commit 81bddfd

Browse files
committed
kafka transport
1 parent c7bb45f commit 81bddfd

9 files changed

Lines changed: 311 additions & 9 deletions

‎pkg/rdkafka/RdKafkaConnectionFactory.php‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ class RdKafkaConnectionFactory implements PsrConnectionFactory
2323
* 'rebalance_cb' => null,
2424
* 'partitioner' => null, // https://arnaud-lb.github.io/php-rdkafka/phpdoc/rdkafka-topicconf.setpartitioner.html
2525
* 'log_level' => null,
26+
* 'commit_async' => false,
2627
* ]
2728
*
2829
* or
@@ -63,7 +64,43 @@ public function createContext()
6364
*/
6465
private function parseDsn($dsn)
6566
{
67+
$dsnConfig = parse_url($dsn);
68+
if (false === $dsnConfig) {
69+
throw new \LogicException(sprintf('Failed to parse DSN "%s"', $dsn));
70+
}
71+
72+
$dsnConfig = array_replace([
73+
'scheme' => null,
74+
'host' => null,
75+
'port' => null,
76+
'user' => null,
77+
'pass' => null,
78+
'path' => null,
79+
'query' => null,
80+
], $dsnConfig);
81+
82+
if ('rdkafka' !== $dsnConfig['scheme']) {
83+
throw new \LogicException(sprintf('The given DSN scheme "%s" is not supported. Could be "rdkafka" only.', $dsnConfig['scheme']));
84+
}
85+
86+
// $query = [];
87+
// if ($dsnConfig['query']) {
88+
// parse_str($dsnConfig['query'], $query);
89+
// }
90+
6691

92+
$broker = $dsnConfig['host'];
93+
if ($dsnConfig['port']) {
94+
$broker .= ':'.$dsnConfig['port'];
95+
}
96+
97+
return [
98+
'global' => [
99+
'group.id' => uniqid('', true),
100+
'metadata.broker.list' => $broker,
101+
'auto.offset.reset' => 'largest',
102+
],
103+
];
67104
}
68105

69106
/**
@@ -74,6 +111,7 @@ private function defaultConfig()
74111
return [
75112
'global' => [
76113
'metadata.broker.list' => 'localhost:9092',
114+
'group.id' => uniqid('', true),
77115
],
78116
];
79117
}

‎pkg/rdkafka/RdKafkaConsumer.php‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
<?php
2+
namespace Enqueue\RdKafka;
3+
4+
use Interop\Queue\InvalidMessageException;
5+
use Interop\Queue\PsrConsumer;
6+
use Interop\Queue\PsrMessage;
7+
use RdKafka\KafkaConsumer;
8+
9+
class RdKafkaConsumer implements PsrConsumer
10+
{
11+
/**
12+
* @var KafkaConsumer
13+
*/
14+
private $consumer;
15+
16+
/**
17+
* @var RdKafkaContext
18+
*/
19+
private $context;
20+
21+
/**
22+
* @var RdKafkaTopic
23+
*/
24+
private $topic;
25+
26+
/**
27+
* @var bool
28+
*/
29+
private $subscribed;
30+
31+
/**
32+
* @var bool
33+
*/
34+
private $commitAsync;
35+
36+
/**
37+
* @param KafkaConsumer $consumer
38+
* @param RdKafkaContext $context
39+
* @param RdKafkaTopic $topic
40+
*/
41+
public function __construct(KafkaConsumer $consumer, RdKafkaContext $context, RdKafkaTopic $topic)
42+
{
43+
$this->consumer = $consumer;
44+
$this->context = $context;
45+
$this->topic = $topic;
46+
$this->subscribed = false;
47+
$this->commitAsync = false;
48+
}
49+
50+
/**
51+
* @return bool
52+
*/
53+
public function isCommitAsync()
54+
{
55+
return $this->commitAsync;
56+
}
57+
58+
/**
59+
* @param bool $async
60+
*/
61+
public function setCommitAsync($async)
62+
{
63+
$this->commitAsync = (bool) $async;
64+
}
65+
66+
/**
67+
* {@inheritdoc}
68+
*/
69+
public function getQueue()
70+
{
71+
return $this->topic;
72+
}
73+
74+
/**
75+
* {@inheritdoc}
76+
*/
77+
public function receive($timeout = 0)
78+
{
79+
$this->consumer->subscribe([$this->topic->getTopicName()]);
80+
81+
if ($timeout > 0) {
82+
$message = $this->doReceive($timeout);
83+
} else {
84+
while (true) {
85+
if ($message = $this->doReceive(500)) {
86+
break;
87+
}
88+
}
89+
}
90+
91+
$this->consumer->unsubscribe();
92+
93+
return $message;
94+
}
95+
96+
/**
97+
* {@inheritdoc}
98+
*/
99+
public function receiveNoWait()
100+
{
101+
return $this->receive(1000);
102+
}
103+
104+
/**
105+
* {@inheritdoc}
106+
*
107+
* @param RdKafkaMessage $message
108+
*/
109+
public function acknowledge(PsrMessage $message)
110+
{
111+
InvalidMessageException::assertMessageInstanceOf($message, RdKafkaMessage::class);
112+
113+
if (false == $message->getKafkaMessage()) {
114+
throw new \LogicException('The message could not be acknowledged because it does not have kafka message set.');
115+
}
116+
117+
if ($this->isCommitAsync()) {
118+
$this->consumer->commitAsync($message->getKafkaMessage());
119+
} else {
120+
$this->consumer->commit($message->getKafkaMessage());
121+
}
122+
}
123+
124+
/**
125+
* {@inheritdoc}
126+
*
127+
* @param RdKafkaMessage $message
128+
*/
129+
public function reject(PsrMessage $message, $requeue = false)
130+
{
131+
$this->acknowledge($message);
132+
133+
if ($requeue) {
134+
$this->context->createProducer()->send($this->topic, $message);
135+
}
136+
}
137+
138+
/**
139+
* @param int $timeout
140+
*
141+
* @return RdKafkaMessage|null
142+
*/
143+
private function doReceive($timeout)
144+
{
145+
$kafkaMessage = $this->consumer->consume($timeout);
146+
147+
switch ($kafkaMessage->err) {
148+
case RD_KAFKA_RESP_ERR__PARTITION_EOF:
149+
case RD_KAFKA_RESP_ERR__TIMED_OUT:
150+
break;
151+
case RD_KAFKA_RESP_ERR_NO_ERROR:
152+
$message = RdKafkaMessage::jsonUnserialize($kafkaMessage->payload);
153+
$message->setKey($kafkaMessage->key);
154+
$message->setPartition($kafkaMessage->partition);
155+
$message->setKafkaMessage($kafkaMessage);
156+
157+
return $message;
158+
default:
159+
throw new \LogicException($kafkaMessage->errstr(), $kafkaMessage->err);
160+
break;
161+
}
162+
}
163+
}

‎pkg/rdkafka/RdKafkaContext.php‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,11 @@
11
<?php
22
namespace Enqueue\RdKafka;
33

4-
use Interop\Queue\PsrConsumer;
4+
use Interop\Queue\InvalidDestinationException;
55
use Interop\Queue\PsrContext;
66
use Interop\Queue\PsrDestination;
7-
use Interop\Queue\PsrMessage;
8-
use Interop\Queue\PsrProducer;
9-
use Interop\Queue\PsrQueue;
10-
use Interop\Queue\PsrTopic;
117
use RdKafka\Conf;
8+
use RdKafka\KafkaConsumer;
129
use RdKafka\Producer;
1310
use RdKafka\TopicConf;
1411

@@ -55,9 +52,14 @@ public function createTopic($topicName)
5552
return new RdKafkaTopic($topicName);
5653
}
5754

55+
/**
56+
* {@inheritdoc}
57+
*
58+
* @return RdKafkaTopic
59+
*/
5860
public function createQueue($queueName)
5961
{
60-
// TODO: Implement createQueue() method.
62+
return new RdKafkaTopic($queueName);
6163
}
6264

6365
/**
@@ -78,14 +80,29 @@ public function createProducer()
7880
return new RdKafkaProducer($this->getProducer());
7981
}
8082

83+
/**
84+
* {@inheritdoc}
85+
*
86+
* @param RdKafkaTopic $destination
87+
*/
8188
public function createConsumer(PsrDestination $destination)
8289
{
83-
// TODO: Implement createConsumer() method.
90+
InvalidDestinationException::assertDestinationInstanceOf($destination, RdKafkaTopic::class);
91+
92+
$consumer = new RdKafkaConsumer(new KafkaConsumer($this->getConf()), $this, $destination);
93+
94+
if (isset($this->config['commit_async'])) {
95+
$consumer->setCommitAsync($this->config['commit_async']);
96+
}
97+
98+
return $consumer;
8499
}
85100

101+
/**
102+
* {@inheritdoc}
103+
*/
86104
public function close()
87105
{
88-
89106
}
90107

91108
/**

‎pkg/rdkafka/RdKafkaMessage.php‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
namespace Enqueue\RdKafka;
33

44
use Interop\Queue\PsrMessage;
5+
use RdKafka\Message;
56

67
class RdKafkaMessage implements PsrMessage, \JsonSerializable
78
{
@@ -35,6 +36,11 @@ class RdKafkaMessage implements PsrMessage, \JsonSerializable
3536
*/
3637
private $key;
3738

39+
/**
40+
* @var Message
41+
*/
42+
private $kafkaMessage;
43+
3844
/**
3945
* @param string $body
4046
* @param array $properties
@@ -242,6 +248,22 @@ public function setKey($key)
242248
$this->key = $key;
243249
}
244250

251+
/**
252+
* @return Message
253+
*/
254+
public function getKafkaMessage()
255+
{
256+
return $this->kafkaMessage;
257+
}
258+
259+
/**
260+
* @param Message $message
261+
*/
262+
public function setKafkaMessage(Message $message)
263+
{
264+
$this->kafkaMessage = $message;
265+
}
266+
245267
/**
246268
* {@inheritdoc}
247269
*/
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<?php
2+
namespace Enqueue\RdKafka\Tests\Spec;
3+
4+
use Enqueue\RdKafka\RdKafkaConnectionFactory;
5+
use Interop\Queue\Spec\PsrConnectionFactorySpec;
6+
7+
class RdKafkaConnectionFactoryTest extends PsrConnectionFactorySpec
8+
{
9+
protected function createConnectionFactory()
10+
{
11+
return new RdKafkaConnectionFactory();
12+
}
13+
}
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?php
2+
namespace Enqueue\RdKafka\Tests\Spec;
3+
4+
use Enqueue\RdKafka\RdKafkaContext;
5+
use Interop\Queue\Spec\PsrContextSpec;
6+
7+
class RdKafkaContextTest extends PsrContextSpec
8+
{
9+
protected function createContext()
10+
{
11+
return new RdKafkaContext([
12+
'global' => [
13+
'group.id' => 'group',
14+
],
15+
]);
16+
}
17+
}

‎pkg/rdkafka/Tests/Spec/RdKafkaMessageTest.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,6 @@ class RdKafkaMessageTest extends PsrMessageSpec
1111
*/
1212
protected function createMessage()
1313
{
14-
return new RdKafkaMessage(self::EXPECTED_TOPIC_NAME);
14+
return new RdKafkaMessage();
1515
}
1616
}
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?php
2+
namespace Enqueue\RdKafka\Tests\Spec;
3+
4+
use Enqueue\RdKafka\RdKafkaConnectionFactory;
5+
use Interop\Queue\Spec\SendToAndReceiveFromTopicSpec;
6+
7+
/**
8+
* @group functional
9+
*/
10+
class RdKafkaSendToAndReceiveFromTopicTest extends SendToAndReceiveFromTopicSpec
11+
{
12+
protected function createContext()
13+
{
14+
return (new RdKafkaConnectionFactory(getenv('RDKAFKA_DSN')))->createContext();
15+
}
16+
}

0 commit comments

Comments
 (0)