Skip to content

Commit 45eba53

Browse files
author
Mike Shultz
authored
Merge pull request #12 from ApeWorX/fix/expiration
fix: handle ExpiredToken from ECS metadata provider credentials
2 parents 70e59bc + 0f94f08 commit 45eba53

3 files changed

Lines changed: 98 additions & 65 deletions

File tree

‎taskiq_sqs/aws.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import asyncio
12
import json
23
import os
34

@@ -10,13 +11,15 @@ class InvalidEnvironment(Exception):
1011
pass
1112

1213

13-
def get_container_credentials():
14+
async def get_container_credentials() -> dict[str, str]:
1415
"""Fetches the ECS task role credentials provided by the metadata service"""
1516
if not (relative_uri := os.environ.get("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")):
1617
raise InvalidEnvironment(
1718
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI not defined. This may not be an ECS container."
1819
)
1920

2021
http = urllib3.PoolManager()
21-
resp = http.request("GET", f"{ECS_CONTAINER_METADATA_URI}{relative_uri}")
22+
resp = await asyncio.to_thread(
23+
http.request, "GET", f"{ECS_CONTAINER_METADATA_URI}{relative_uri}"
24+
)
2225
return json.loads(resp.data)

‎taskiq_sqs/broker.py‎

Lines changed: 91 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import asyncio
66
import logging
7+
from collections import defaultdict
78
from datetime import datetime, timezone
89
from typing import TYPE_CHECKING, AsyncGenerator, Callable, Optional, Union
910

@@ -19,6 +20,8 @@
1920
from taskiq_sqs.aws import get_container_credentials
2021

2122
if TYPE_CHECKING:
23+
from typing import Mapping
24+
2225
from mypy_boto3_sqs.service_resource import Queue, SQSServiceResource
2326

2427
logger = logging.getLogger(__name__)
@@ -46,22 +49,14 @@ def __init__(
4649
if not sqs_queue_url or not sqs_queue_url.startswith("http"):
4750
raise BrokerError("A valid SQS Queue URL is required")
4851

49-
creds = dict()
5052
# NOTE: This bypasses the normal order of operations for boto3 auth and
5153
# goes straight to using the ECS role creds from the metadata
5254
# service. This can be useful in edge cases where there are higher
5355
# priority credentials you do not want to use for this service.
54-
if force_ecs_container_credentials:
55-
creds = get_container_credentials()
56-
56+
self.force_ecs_container_credentials = force_ecs_container_credentials
57+
self.sqs_region_override = sqs_region_override
5758
self.sqs_queue_url = sqs_queue_url
58-
self._sqs: SQSServiceResource = boto3.resource(
59-
"sqs",
60-
region_name=sqs_region_override,
61-
aws_access_key_id=creds.get("AccessKeyId"),
62-
aws_secret_access_key=creds.get("SecretAccessKey"),
63-
aws_session_token=creds.get("Token"),
64-
)
59+
self._sqs: SQSServiceResource | None = None
6560
self._sqs_queue: Optional[Queue] = None
6661

6762
if max_number_of_messages > 10:
@@ -70,16 +65,43 @@ def __init__(
7065
self.wait_time_seconds = max(wait_time_seconds, 0)
7166
self.max_number_of_messages = max(max_number_of_messages, 1)
7267

68+
@property
69+
def _sqs_credentials_expired(self):
70+
return self._creds_expiration and self._creds_expiration < datetime.now(
71+
tz=timezone.utc
72+
)
73+
74+
async def _sqs_client(self) -> SQSServiceResource:
75+
if self._sqs and not self._sqs_credentials_expired:
76+
return self._sqs
77+
78+
creds: Mapping[str, str] = defaultdict(None)
79+
80+
if self.force_ecs_container_credentials:
81+
creds = await get_container_credentials()
82+
# NOTE: This is probably not an optional prop in the response
83+
if creds.get("Expiration"):
84+
self._creds_expiration = datetime.fromisoformat(creds["Expiration"])
85+
86+
return boto3.resource(
87+
"sqs",
88+
region_name=self.sqs_region_override,
89+
aws_access_key_id=creds.get("AccessKeyId"),
90+
aws_secret_access_key=creds.get("SecretAccessKey"),
91+
aws_session_token=creds.get("Token"),
92+
)
93+
7394
async def _get_queue(self) -> Queue:
74-
queue_name = self.sqs_queue_url.split("/")[-1]
95+
if self._sqs_queue and not self._sqs_credentials_expired:
96+
return self._sqs_queue
7597

76-
if not self._sqs_queue:
77-
self._sqs_queue = await asyncify(self._sqs.get_queue_by_name)(
78-
QueueName=queue_name
79-
)
98+
sqs = await self._sqs_client()
99+
self._sqs_queue = await asyncify(sqs.get_queue_by_name)(
100+
QueueName=self.sqs_queue_url.split("/")[-1]
101+
)
80102

81-
if not self._sqs_queue:
82-
raise Exception("SQS Queue not found")
103+
if not self._sqs_queue:
104+
raise Exception("SQS Queue not found")
83105

84106
return self._sqs_queue
85107

@@ -136,53 +158,60 @@ async def listen(self) -> AsyncGenerator[Union[bytes, AckableMessage], None]:
136158
:return: nothing.
137159
"""
138160

139-
queue = await self._get_queue()
140-
141161
# TODO: Consider using AckableMessage and confirm with the queue to reduce lost messages
142162
while True:
143163
no_backoff = False
144-
145-
for message in await asyncify(queue.receive_messages)(
146-
MessageAttributeNames=[".*"],
147-
# If there's competition on this queue (multiple processes of workers pulling from
148-
# the same queue), and processing takes longer than the visibility timeout, multiple
149-
# workers may end up processing the same message.
150-
MaxNumberOfMessages=self.max_number_of_messages,
151-
# Use long poling.
152-
WaitTimeSeconds=self.wait_time_seconds,
153-
):
154-
try:
155-
if message.message_attributes:
156-
# if expiry was set as a message attribute, respect it
157-
if expiry_typed := message.message_attributes.get("expiry"):
158-
expiry = int(expiry_typed.get("StringValue", 0))
159-
now = stamp()
160-
if 0 < expiry < now:
161-
logger.warn(
162-
f"Message expired {now - expiry} seconds ago. Skipping."
163-
)
164-
await asyncify(message.delete)()
165-
no_backoff = True
166-
continue
167-
except TypeError:
168-
# Ignore weird expiries. Not critical.
169-
pass
170-
171-
yield message.body.encode("utf-8")
172-
173-
try:
174-
await asyncify(message.delete)()
175-
except ClientError as err:
176-
if "receipt handle has expired" in str(err):
177-
# while not ideal, we shouldn't die on this
178-
logger.error(
179-
"Message receipt handle has expired. This could indicate duplicate"
180-
"processing or tasks being processed late."
181-
)
182-
else:
183-
raise err
184-
185-
no_backoff = True
164+
queue = await self._get_queue()
165+
166+
try:
167+
for message in await asyncify(queue.receive_messages)(
168+
MessageAttributeNames=[".*"],
169+
# If there's competition on this queue (multiple processes of workers pulling
170+
# from the same queue), and processing takes longer than the visibility timeout,
171+
# multiple workers may end up processing the same message.
172+
MaxNumberOfMessages=self.max_number_of_messages,
173+
# Use long poling.
174+
WaitTimeSeconds=self.wait_time_seconds,
175+
):
176+
try:
177+
if message.message_attributes:
178+
# if expiry was set as a message attribute, respect it
179+
if expiry_typed := message.message_attributes.get("expiry"):
180+
expiry = int(expiry_typed.get("StringValue", 0))
181+
now = stamp()
182+
if 0 < expiry < now:
183+
logger.warn(
184+
f"Message expired {now - expiry} seconds ago. Skipping."
185+
)
186+
await asyncify(message.delete)()
187+
no_backoff = True
188+
continue
189+
except TypeError:
190+
# Ignore weird expiries. Not critical.
191+
pass
192+
193+
yield message.body.encode("utf-8")
194+
195+
try:
196+
await asyncify(message.delete)()
197+
except ClientError as err:
198+
if "receipt handle has expired" in str(err):
199+
# while not ideal, we shouldn't die on this
200+
logger.error(
201+
"Message receipt handle has expired. This could indicate duplicate"
202+
"processing or tasks being processed late."
203+
)
204+
else:
205+
raise err
206+
207+
no_backoff = True
208+
except ClientError as err:
209+
# Creds will get refreshed when _get_queue() is called again
210+
if "ExpiredToken" in str(err):
211+
logger.warning("ECS credentials expired.")
212+
continue
213+
else:
214+
raise err
186215

187216
sleepdur = 0.01 if no_backoff else 1
188217
logger.debug(f"No messages on queue. Broker is sleeping for {sleepdur}s...")

‎tests/test_broker.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,6 @@ def test_init():
77
broker.sqs_queue_url
88
== "https://sqs.us-west-2.amazonaws.com/123456789012/queue-name"
99
)
10-
assert broker._sqs is not None
10+
assert broker.force_ecs_container_credentials is False
11+
assert broker.sqs_region_override is None
1112
assert broker._sqs_queue is None

0 commit comments

Comments
 (0)