Skip to content

Commit c641c2a

Browse files
committed
Introduce redelivery support based on visibility approach
1 parent e420a55 commit c641c2a

7 files changed

Lines changed: 209 additions & 160 deletions

File tree

‎pkg/dbal/DbalConsumer.php‎

Lines changed: 89 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,18 @@
66

77
use Doctrine\DBAL\Connection;
88
use Doctrine\DBAL\Types\Type;
9+
use function GuzzleHttp\Psr7\str;
910
use Interop\Queue\Consumer;
1011
use Interop\Queue\Exception\InvalidMessageException;
12+
use Interop\Queue\Impl\ConsumerPollingTrait;
1113
use Interop\Queue\Message;
1214
use Interop\Queue\Queue;
15+
use Ramsey\Uuid\Uuid;
1316

1417
class DbalConsumer implements Consumer
1518
{
19+
use ConsumerPollingTrait;
20+
1621
/**
1722
* @var DbalContext
1823
*/
@@ -29,33 +34,37 @@ class DbalConsumer implements Consumer
2934
private $queue;
3035

3136
/**
37+
* Default 20 minutes in milliseconds.
38+
*
3239
* @var int
3340
*/
34-
private $pollingInterval;
41+
private $redeliveryDelay;
3542

3643
public function __construct(DbalContext $context, DbalDestination $queue)
3744
{
3845
$this->context = $context;
3946
$this->queue = $queue;
4047
$this->dbal = $this->context->getDbalConnection();
4148

42-
$this->pollingInterval = 1000;
49+
$this->redeliveryDelay = 1200000;
4350
}
4451

4552
/**
46-
* Polling interval is in milliseconds.
53+
* Get interval between retry failed messages in milliseconds.
4754
*/
48-
public function setPollingInterval(int $interval): void
55+
public function getRedeliveryDelay(): int
4956
{
50-
$this->pollingInterval = $interval;
57+
return $this->redeliveryDelay;
5158
}
5259

5360
/**
54-
* Get polling interval in milliseconds.
61+
* Interval between retry failed messages in seconds.
5562
*/
56-
public function getPollingInterval(): int
63+
public function setRedeliveryDelay(int $redeliveryDelay): self
5764
{
58-
return $this->pollingInterval;
65+
$this->redeliveryDelay = $redeliveryDelay;
66+
67+
return $this;
5968
}
6069

6170
/**
@@ -66,41 +75,67 @@ public function getQueue(): Queue
6675
return $this->queue;
6776
}
6877

69-
public function receive(int $timeout = 0): ?Message
78+
public function receiveNoWait(): ?Message
7079
{
71-
$timeout /= 1000;
72-
$startAt = microtime(true);
80+
$this->redeliverMessages();
81+
82+
$this->dbal->beginTransaction();
83+
try {
84+
$now = (int) time();
85+
$redeliveryDelay = $this->getRedeliveryDelay() / 1000; // milliseconds to seconds
86+
$deliveryId = (string) Uuid::uuid1();
7387

74-
while (true) {
75-
$message = $this->receiveMessage();
88+
// get top message from the queue
89+
$message = $this->fetchMessage($now);
7690

77-
if ($message) {
78-
return $message;
79-
}
91+
if (null == $message) {
92+
$this->dbal->commit();
8093

81-
if ($timeout && (microtime(true) - $startAt) >= $timeout) {
8294
return null;
8395
}
8496

85-
usleep($this->pollingInterval * 1000);
97+
// mark message as delivered to consumer
98+
$this->dbal->createQueryBuilder()
99+
->update($this->context->getTableName())
100+
->set('delivery_id', ':deliveryId')
101+
->set('redeliver_after', ':redeliverAfter')
102+
->andWhere('id = :id')
103+
->setParameter('id', $message['id'], Type::GUID)
104+
->setParameter('deliveryId', $deliveryId, Type::STRING)
105+
->setParameter('redeliverAfter', $now + $redeliveryDelay, Type::BIGINT)
106+
->execute()
107+
;
108+
109+
$dbalMessage = $this->dbal->createQueryBuilder()
110+
->select('*')
111+
->from($this->context->getTableName())
112+
->andWhere('delivery_id = :deliveryId')
113+
->setParameter('deliveryId', $deliveryId, Type::STRING)
114+
->setMaxResults(1)
115+
->execute()
116+
->fetch()
117+
;
86118

87-
if ($timeout && (microtime(true) - $startAt) >= $timeout) {
88-
return null;
119+
$this->dbal->commit();
120+
121+
if ($message->isRedelivered() || empty($dbalMessage['time_to_live']) || $dbalMessage['time_to_live'] > time()) {
122+
return $this->context->convertMessage($dbalMessage);
89123
}
90-
}
91-
}
92124

93-
public function receiveNoWait(): ?Message
94-
{
95-
return $this->receiveMessage();
125+
return null;
126+
} catch (\Exception $e) {
127+
$this->dbal->rollBack();
128+
129+
throw $e;
130+
}
96131
}
97132

98133
/**
99134
* @param DbalMessage $message
100135
*/
101136
public function acknowledge(Message $message): void
102137
{
103-
// does nothing
138+
$this->deleteMessage($message->getDeliveryId());
104139
}
105140

106141
/**
@@ -115,102 +150,51 @@ public function reject(Message $message, bool $requeue = false): void
115150

116151
return;
117152
}
153+
154+
$this->deleteMessage($message->getDeliveryId());
118155
}
119156

120-
protected function receiveMessage(): ?DbalMessage
157+
private function deleteMessage(?string $deliveryId): void
121158
{
122-
$this->dbal->beginTransaction();
123-
try {
124-
$now = time();
125-
126-
$dbalMessage = $this->fetchPrioritizedMessage($now) ?: $dbalMessage = $this->fetchMessage($now);
127-
if (false == $dbalMessage) {
128-
$this->dbal->commit();
129-
130-
return null;
131-
}
132-
133-
// remove message
134-
$affectedRows = $this->dbal->delete($this->context->getTableName(), ['id' => $dbalMessage['id']], [
135-
'id' => Type::GUID,
136-
]);
137-
138-
if (1 !== $affectedRows) {
139-
throw new \LogicException(sprintf('Expected record was removed but it is not. id: "%s"', $dbalMessage['id']));
140-
}
141-
142-
$this->dbal->commit();
143-
144-
if (empty($dbalMessage['time_to_live']) || ($dbalMessage['time_to_live'] / 1000) > microtime(true)) {
145-
return $this->context->convertMessage($dbalMessage);
146-
}
147-
148-
return null;
149-
} catch (\Exception $e) {
150-
$this->dbal->rollBack();
151-
152-
throw $e;
153-
}
159+
$this->dbal->delete(
160+
$this->context->getTableName(),
161+
['delivery_id' => $deliveryId],
162+
['delivery_id' => Type::STRING]
163+
);
154164
}
155165

156-
private function fetchPrioritizedMessage(int $now): ?array
166+
private function fetchMessage(int $now): ?array
157167
{
158-
$query = $this->dbal->createQueryBuilder();
159-
$query
168+
$result = $this->dbal->createQueryBuilder()
160169
->select('*')
161170
->from($this->context->getTableName())
171+
->andWhere('delivery_id IS NULL')
162172
->andWhere('queue = :queue')
163-
->andWhere('priority IS NOT NULL')
164-
->andWhere('(delayed_until IS NULL OR delayed_until <= :delayedUntil)')
165-
->addOrderBy('published_at', 'asc')
173+
->andWhere('delayed_until IS NULL OR delayed_until <= :delayedUntil')
166174
->addOrderBy('priority', 'desc')
175+
->addOrderBy('published_at', 'asc')
176+
->setParameter('queue', $this->queue->getQueueName(), Type::STRING)
177+
->setParameter('delayedUntil', $now, Type::BIGINT)
167178
->setMaxResults(1)
179+
->execute()
180+
->fetch()
168181
;
169182

170-
$sql = $query->getSQL().' '.$this->dbal->getDatabasePlatform()->getWriteLockSQL();
171-
172-
$result = $this->dbal->executeQuery(
173-
$sql,
174-
[
175-
'queue' => $this->queue->getQueueName(),
176-
'delayedUntil' => $now,
177-
],
178-
[
179-
'queue' => Type::STRING,
180-
'delayedUntil' => Type::INTEGER,
181-
]
182-
)->fetch();
183-
184183
return $result ?: null;
185184
}
186185

187-
private function fetchMessage(int $now): ?array
186+
private function redeliverMessages(): void
188187
{
189-
$query = $this->dbal->createQueryBuilder();
190-
$query
191-
->select('*')
192-
->from($this->context->getTableName())
193-
->andWhere('queue = :queue')
194-
->andWhere('priority IS NULL')
195-
->andWhere('(delayed_until IS NULL OR delayed_until <= :delayedUntil)')
196-
->addOrderBy('published_at', 'asc')
197-
->setMaxResults(1)
188+
$this->dbal->createQueryBuilder()
189+
->update($this->context->getTableName())
190+
->set('delivery_id', ':deliveryId')
191+
->set('redelivered', ':redelivered')
192+
->andWhere('delivery_id IS NOT NULL')
193+
->andWhere('redeliver_after < :now')
194+
->setParameter(':now', (int) time(), Type::BIGINT)
195+
->setParameter('deliveryId', null, Type::STRING)
196+
->setParameter('redelivered', true, Type::BOOLEAN)
197+
->execute()
198198
;
199-
200-
$sql = $query->getSQL().' '.$this->dbal->getDatabasePlatform()->getWriteLockSQL();
201-
202-
$result = $this->dbal->executeQuery(
203-
$sql,
204-
[
205-
'queue' => $this->queue->getQueueName(),
206-
'delayedUntil' => $now,
207-
],
208-
[
209-
'queue' => Type::STRING,
210-
'delayedUntil' => Type::INTEGER,
211-
]
212-
)->fetch();
213-
214-
return $result ?: null;
215199
}
216200
}

‎pkg/dbal/DbalContext.php‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,10 @@ public function createConsumer(Destination $destination): Consumer
116116
$consumer->setPollingInterval($this->config['polling_interval']);
117117
}
118118

119+
if (isset($this->config['redelivery_delay'])) {
120+
$consumer->setRedeliveryDelay($this->config['redelivery_delay']);
121+
}
122+
119123
return $consumer;
120124
}
121125

@@ -133,6 +137,7 @@ public function createSubscriptionConsumer(): SubscriptionConsumer
133137
*/
134138
public function convertMessage(array $dbalMessage): DbalMessage
135139
{
140+
/** @var DbalMessage $dbalMessageObj */
136141
$dbalMessageObj = $this->createMessage(
137142
$dbalMessage['body'],
138143
$dbalMessage['properties'] ? JSON::decode($dbalMessage['properties']) : [],
@@ -148,6 +153,12 @@ public function convertMessage(array $dbalMessage): DbalMessage
148153
if (isset($dbalMessage['published_at'])) {
149154
$dbalMessageObj->setPublishedAt((int) $dbalMessage['published_at']);
150155
}
156+
if (isset($dbalMessage['delivery_id'])) {
157+
$dbalMessageObj->setDeliveryId((string) $dbalMessage['delivery_id']);
158+
}
159+
if (isset($dbalMessage['redeliver_after'])) {
160+
$dbalMessageObj->setRedeliverAfter((int) $dbalMessage['redeliver_after']);
161+
}
151162

152163
return $dbalMessageObj;
153164
}
@@ -212,13 +223,18 @@ public function createDataBaseTable(): void
212223
$table->addColumn('priority', Type::SMALLINT, ['notnull' => false]);
213224
$table->addColumn('delayed_until', Type::BIGINT, ['notnull' => false]);
214225
$table->addColumn('time_to_live', Type::BIGINT, ['notnull' => false]);
226+
$table->addColumn('delivery_id', Type::STRING, ['notnull' => false]);
227+
$table->addColumn('redeliver_after', Type::BIGINT, ['notnull' => false]);
215228

216229
$table->setPrimaryKey(['id']);
217230
$table->addIndex(['published_at']);
218231
$table->addIndex(['queue']);
219232
$table->addIndex(['priority']);
220233
$table->addIndex(['delayed_until']);
221234
$table->addIndex(['priority', 'published_at']);
235+
$table->addIndex(['redeliver_after']);
236+
$table->addUniqueIndex(['delivery_id']);
237+
$table->addIndex(['delivery_id']);
222238

223239
$sm->createTable($table);
224240
}

‎pkg/dbal/DbalMessage.php‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,21 @@ class DbalMessage implements Message
3838
*/
3939
private $deliveryDelay;
4040

41+
/**
42+
* @var int seconds
43+
*/
44+
private $redeliverAfter;
45+
4146
/**
4247
* @var int milliseconds
4348
*/
4449
private $timeToLive;
4550

51+
/**
52+
* @var null|string
53+
*/
54+
private $deliveryId;
55+
4656
/**
4757
* Milliseconds, for example 15186054527288.
4858
*
@@ -65,6 +75,8 @@ public function __construct(string $body = '', array $properties = [], array $he
6575
$this->redelivered = false;
6676
$this->priority = null;
6777
$this->deliveryDelay = null;
78+
$this->deliveryId = null;
79+
$this->redeliverAfter = null;
6880
}
6981

7082
public function setBody(string $body): void
@@ -208,6 +220,26 @@ public function setTimestamp(int $timestamp = null): void
208220
$this->setHeader('timestamp', $timestamp);
209221
}
210222

223+
public function getDeliveryId(): ?string
224+
{
225+
return $this->deliveryId;
226+
}
227+
228+
public function setDeliveryId(?string $deliveryId = null): void
229+
{
230+
$this->deliveryId = $deliveryId;
231+
}
232+
233+
public function getRedeliverAfter(): int
234+
{
235+
return $this->redeliverAfter;
236+
}
237+
238+
public function setRedeliverAfter(int $redeliverAfter = null): void
239+
{
240+
$this->redeliverAfter = $redeliverAfter;
241+
}
242+
211243
public function getPublishedAt(): ?int
212244
{
213245
return $this->publishedAt;

0 commit comments

Comments
 (0)