Skip to content

Commit dee836c

Browse files
authored
Merge pull request #581 from rosamarsky/dbal-subscription-consumer-improvements
[dbal] Introduce redelivery support based on visibility approach.
2 parents 16a9490 + 239161b commit dee836c

9 files changed

Lines changed: 430 additions & 193 deletions

File tree

‎docs/transport/dbal.md‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,14 @@
22

33
The transport uses [Doctrine DBAL](http://docs.doctrine-project.org/projects/doctrine-dbal/en/latest/) library and SQL like server as a broker.
44
It creates a table there. Pushes and pops messages to\from that table.
5-
6-
**Limitations** It works only in auto ack mode hence If consumer crashes the message is lost.
75

86
* [Installation](#installation)
97
* [Init database](#init-database)
108
* [Create context](#create-context)
119
* [Send message to topic](#send-message-to-topic)
1210
* [Send message to queue](#send-message-to-queue)
11+
* [Send expiration message](#send-expiration-message)
12+
* [Send delayed message](#send-delayed-message)
1313
* [Consume message](#consume-message)
1414
* [Subscription consumer](#subscription-consumer)
1515

@@ -90,6 +90,38 @@ $message = $context->createMessage('Hello world!');
9090
$context->createProducer()->send($fooQueue, $message);
9191
```
9292

93+
## Send expiration message
94+
95+
```php
96+
<?php
97+
/** @var \Enqueue\Dbal\DbalContext $psrContext */
98+
/** @var \Enqueue\Dbal\DbalDestination $fooQueue */
99+
100+
$message = $psrContext->createMessage('Hello world!');
101+
102+
$psrContext->createProducer()
103+
->setTimeToLive(60000) // 60 sec
104+
//
105+
->send($fooQueue, $message)
106+
;
107+
```
108+
109+
## Send delayed message
110+
111+
```php
112+
<?php
113+
/** @var \Enqueue\Dbal\DbalContext $psrContext */
114+
/** @var \Enqueue\Dbal\DbalDestination $fooQueue */
115+
116+
$message = $psrContext->createMessage('Hello world!');
117+
118+
$psrContext->createProducer()
119+
->setDeliveryDelay(5000) // 5 sec
120+
//
121+
->send($fooQueue, $message)
122+
;
123+
````
124+
93125
## Consume message:
94126

95127
```php
@@ -102,6 +134,9 @@ $consumer = $context->createConsumer($fooQueue);
102134
$message = $consumer->receive();
103135

104136
// process a message
137+
138+
$consumer->acknowledge($message);
139+
//$consumer->reject($message);
105140
```
106141

107142
## Subscription consumer

‎pkg/dbal/DbalConsumer.php‎

Lines changed: 48 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,15 @@
88
use Doctrine\DBAL\Types\Type;
99
use Interop\Queue\Consumer;
1010
use Interop\Queue\Exception\InvalidMessageException;
11+
use Interop\Queue\Impl\ConsumerPollingTrait;
1112
use Interop\Queue\Message;
1213
use Interop\Queue\Queue;
1314

1415
class DbalConsumer implements Consumer
1516
{
17+
use ConsumerPollingTrait,
18+
DbalConsumerHelperTrait;
19+
1620
/**
1721
* @var DbalContext
1822
*/
@@ -29,33 +33,37 @@ class DbalConsumer implements Consumer
2933
private $queue;
3034

3135
/**
36+
* Default 20 minutes in milliseconds.
37+
*
3238
* @var int
3339
*/
34-
private $pollingInterval;
40+
private $redeliveryDelay;
3541

3642
public function __construct(DbalContext $context, DbalDestination $queue)
3743
{
3844
$this->context = $context;
3945
$this->queue = $queue;
4046
$this->dbal = $this->context->getDbalConnection();
4147

42-
$this->pollingInterval = 1000;
48+
$this->redeliveryDelay = 1200000;
4349
}
4450

4551
/**
46-
* Polling interval is in milliseconds.
52+
* Get interval between retry failed messages in milliseconds.
4753
*/
48-
public function setPollingInterval(int $interval): void
54+
public function getRedeliveryDelay(): int
4955
{
50-
$this->pollingInterval = $interval;
56+
return $this->redeliveryDelay;
5157
}
5258

5359
/**
54-
* Get polling interval in milliseconds.
60+
* Get interval between retrying failed messages in milliseconds.
5561
*/
56-
public function getPollingInterval(): int
62+
public function setRedeliveryDelay(int $redeliveryDelay): self
5763
{
58-
return $this->pollingInterval;
64+
$this->redeliveryDelay = $redeliveryDelay;
65+
66+
return $this;
5967
}
6068

6169
/**
@@ -66,41 +74,31 @@ public function getQueue(): Queue
6674
return $this->queue;
6775
}
6876

69-
public function receive(int $timeout = 0): ?Message
77+
public function receiveNoWait(): ?Message
7078
{
71-
$timeout /= 1000;
72-
$startAt = microtime(true);
73-
74-
while (true) {
75-
$message = $this->receiveMessage();
76-
77-
if ($message) {
78-
return $message;
79-
}
80-
81-
if ($timeout && (microtime(true) - $startAt) >= $timeout) {
82-
return null;
83-
}
79+
$redeliveryDelay = $this->getRedeliveryDelay() / 1000; // milliseconds to seconds
8480

85-
usleep($this->pollingInterval * 1000);
81+
$this->removeExpiredMessages();
82+
$this->redeliverMessages();
8683

87-
if ($timeout && (microtime(true) - $startAt) >= $timeout) {
88-
return null;
84+
// get top message from the queue
85+
if ($message = $this->fetchMessage([$this->queue->getQueueName()], $redeliveryDelay)) {
86+
if ($message['redelivered'] || empty($message['time_to_live']) || $message['time_to_live'] > time()) {
87+
return $this->getContext()->convertMessage($message);
8988
}
9089
}
91-
}
9290

93-
public function receiveNoWait(): ?Message
94-
{
95-
return $this->receiveMessage();
91+
return null;
9692
}
9793

9894
/**
9995
* @param DbalMessage $message
10096
*/
10197
public function acknowledge(Message $message): void
10298
{
103-
// does nothing
99+
InvalidMessageException::assertMessageInstanceOf($message, DbalMessage::class);
100+
101+
$this->deleteMessage($message->getDeliveryId());
104102
}
105103

106104
/**
@@ -111,106 +109,37 @@ public function reject(Message $message, bool $requeue = false): void
111109
InvalidMessageException::assertMessageInstanceOf($message, DbalMessage::class);
112110

113111
if ($requeue) {
114-
$this->context->createProducer()->send($this->queue, $message);
112+
$message = clone $message;
113+
$message->setRedelivered(false);
114+
115+
$this->getContext()->createProducer()->send($this->queue, $message);
115116

116117
return;
117118
}
119+
120+
$this->deleteMessage($message->getDeliveryId());
118121
}
119122

120-
protected function receiveMessage(): ?DbalMessage
123+
protected function getContext(): DbalContext
121124
{
122-
$this->dbal->beginTransaction();
123-
try {
124-
$now = time();
125-
126-
$dbalMessage = $this->fetchPrioritizedMessage($now) ?: $dbalMessage = $this->fetchMessage($now);
127-
if (false == $dbalMessage) {
128-
$this->dbal->commit();
129-
130-
return null;
131-
}
132-
133-
// remove message
134-
$affectedRows = $this->dbal->delete($this->context->getTableName(), ['id' => $dbalMessage['id']], [
135-
'id' => Type::GUID,
136-
]);
137-
138-
if (1 !== $affectedRows) {
139-
throw new \LogicException(sprintf('Expected record was removed but it is not. id: "%s"', $dbalMessage['id']));
140-
}
141-
142-
$this->dbal->commit();
143-
144-
if (empty($dbalMessage['time_to_live']) || ($dbalMessage['time_to_live'] / 1000) > microtime(true)) {
145-
return $this->context->convertMessage($dbalMessage);
146-
}
147-
148-
return null;
149-
} catch (\Exception $e) {
150-
$this->dbal->rollBack();
151-
152-
throw $e;
153-
}
125+
return $this->context;
154126
}
155127

156-
private function fetchPrioritizedMessage(int $now): ?array
128+
protected function getConnection(): Connection
157129
{
158-
$query = $this->dbal->createQueryBuilder();
159-
$query
160-
->select('*')
161-
->from($this->context->getTableName())
162-
->andWhere('queue = :queue')
163-
->andWhere('priority IS NOT NULL')
164-
->andWhere('(delayed_until IS NULL OR delayed_until <= :delayedUntil)')
165-
->addOrderBy('published_at', 'asc')
166-
->addOrderBy('priority', 'desc')
167-
->setMaxResults(1)
168-
;
169-
170-
$sql = $query->getSQL().' '.$this->dbal->getDatabasePlatform()->getWriteLockSQL();
171-
172-
$result = $this->dbal->executeQuery(
173-
$sql,
174-
[
175-
'queue' => $this->queue->getQueueName(),
176-
'delayedUntil' => $now,
177-
],
178-
[
179-
'queue' => Type::STRING,
180-
'delayedUntil' => Type::INTEGER,
181-
]
182-
)->fetch();
183-
184-
return $result ?: null;
130+
return $this->dbal;
185131
}
186132

187-
private function fetchMessage(int $now): ?array
133+
private function deleteMessage(string $deliveryId): void
188134
{
189-
$query = $this->dbal->createQueryBuilder();
190-
$query
191-
->select('*')
192-
->from($this->context->getTableName())
193-
->andWhere('queue = :queue')
194-
->andWhere('priority IS NULL')
195-
->andWhere('(delayed_until IS NULL OR delayed_until <= :delayedUntil)')
196-
->addOrderBy('published_at', 'asc')
197-
->setMaxResults(1)
198-
;
199-
200-
$sql = $query->getSQL().' '.$this->dbal->getDatabasePlatform()->getWriteLockSQL();
201-
202-
$result = $this->dbal->executeQuery(
203-
$sql,
204-
[
205-
'queue' => $this->queue->getQueueName(),
206-
'delayedUntil' => $now,
207-
],
208-
[
209-
'queue' => Type::STRING,
210-
'delayedUntil' => Type::INTEGER,
211-
]
212-
)->fetch();
213-
214-
return $result ?: null;
135+
if (empty($deliveryId)) {
136+
throw new \LogicException(sprintf('Expected record was removed but it is not. Delivery id: "%s"', $deliveryId));
137+
}
138+
139+
$this->getConnection()->delete(
140+
$this->getContext()->getTableName(),
141+
['delivery_id' => $deliveryId],
142+
['delivery_id' => Type::STRING]
143+
);
215144
}
216145
}

0 commit comments

Comments
 (0)