fix: honor Retry-After without shortening backoff

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.43.2] - 2026-09-07
+
+### Fixed
+
+- Retryable HTTP errors (including AI Mode capacity `503`s) now honor
+  `Retry-After` seconds or HTTP-dates without shortening exponential backoff.
+  Missing or invalid headers retain backoff; transport retries and retry
+  limits are unchanged.
+- HTTP-date or malformed `Retry-After` headers on `429` responses no longer
+  raise `ValueError` instead of `RateLimitError`. Invalid values retain the
+  existing 60-second metadata fallback; `429` is still not retried by default.
+- Align the exported SDK version and User-Agent with the package version.
+
 ## [0.43.0] - 2026-09-04
 
 ### Added

README.md

@@ -177,8 +177,11 @@ client = ScrapeBadger(config=config)
 
 The SDK automatically retries requests that fail with 500, 502, 503, or 504 status
 codes, as well as transport-level failures (timeouts, network errors, dropped
-connections), using exponential backoff (1s, 2s, 4s, 8s, ...). Each retry logs a
-warning:
+connections), using exponential backoff (1s, 2s, 4s, 8s, ...). Retryable HTTP
+responses with a valid `Retry-After` header wait at least that long, whether
+given as seconds or an HTTP-date, without shortening exponential backoff.
+Missing or invalid headers keep the backoff, and `max_retries` still limits
+retry attempts. Each retry logs a warning:
 
 ```
 ⚠ 503 Service Unavailable — retrying in 4s (attempt 3/10)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapebadger"
-version = "0.43.1"
+version = "0.43.2"
 description = "Official Python SDK for ScrapeBadger - Async web scraping APIs for Twitter and more"
 readme = "README.md"
 license = { text = "MIT" }

src/scrapebadger/__init__.py

@@ -1186,7 +1186,7 @@ async def main():
     ZestimateHistoryPoint as ZillowZestimateHistoryPoint,
 )
 
-__version__ = "0.43.0"
+__version__ = "0.43.2"
 
 __all__ = [
     # TikTok core models

src/scrapebadger/_internal/client.py

@@ -4,6 +4,10 @@
 
 import asyncio
 import logging
+import math
+import time
+from datetime import timezone
+from email.utils import parsedate_to_datetime
 from typing import TYPE_CHECKING, Any, TypeVar
 
 import httpx
@@ -40,10 +44,29 @@
 T = TypeVar("T")
 
 # User agent for SDK requests
-SDK_VERSION = "0.43.0"
+SDK_VERSION = "0.43.2"
 USER_AGENT = f"scrapebadger-python/{SDK_VERSION}"
 
 
+def _retry_after_seconds(response: httpx.Response, default: int) -> int:
+    """Read delay-seconds or an HTTP-date; malformed headers use the caller's fallback."""
+    value = response.headers.get("Retry-After", "").strip()
+    try:
+        if value.isascii() and value.isdecimal():
+            delay = int(value)
+        else:
+            retry_at = parsedate_to_datetime(value)
+            # The obsolete asctime HTTP-date form has no explicit timezone.
+            if retry_at.tzinfo is None:
+                retry_at = retry_at.replace(tzinfo=timezone.utc)
+            delay = max(0, math.ceil(retry_at.timestamp() - time.time()))
+        if math.isfinite(delay):
+            return delay
+    except (OverflowError, TypeError, ValueError):
+        pass
+    return default
+
+
 class BaseClient:
     """Base HTTP client with retry logic and error handling.
 
@@ -164,7 +187,7 @@ def _handle_error_response(
                 limit=data.get("limit"),
                 remaining=data.get("remaining"),
                 reset_at=data.get("reset_at"),
-                retry_after=int(response.headers.get("Retry-After", 60)),
+                retry_after=_retry_after_seconds(response, default=60),
                 tier=data.get("tier"),
             )
 
@@ -250,7 +273,7 @@ async def _request_with_retry(
 
                     # Retry on configured status codes
                     if attempt < self._config.max_retries:
-                        delay = 2**attempt
+                        delay = max(2**attempt, _retry_after_seconds(response, default=0))
                         logger.warning(
                             "⚠ %s %s — retrying in %ss (attempt %d/%d)",
                             response.status_code,

tests/test_retry_warnings.py

@@ -2,15 +2,19 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
-from unittest.mock import AsyncMock, MagicMock, patch
+from email.utils import formatdate
+from unittest.mock import AsyncMock, MagicMock, call, patch
 
 import httpx
 import pytest
+import respx
 
+from scrapebadger import ScrapeBadger
 from scrapebadger._internal.client import BaseClient
 from scrapebadger._internal.config import ClientConfig
-from scrapebadger._internal.exceptions import ScrapeBadgerError
+from scrapebadger._internal.exceptions import RateLimitError, ScrapeBadgerError, ServerError
 
 
 @pytest.fixture
@@ -43,6 +47,147 @@ def test_with_overrides_can_change_retries(self, config_default: ClientConfig) -
         assert new_cfg.max_retries == 3
 
 
+class TestRetryAfter:
+    """Retry server capacity errors without ignoring the server's waiting period."""
+
+    @pytest.mark.parametrize(
+        ("value", "expected"),
+        [
+            ("5", 5),
+            ("0", 1),
+            (" 5 ", 5),
+            (formatdate(1_800_000_005, usegmt=True), 5),
+            ("Fri Jan 15 08:00:05 2027", 5),
+            (formatdate(1_799_999_999, usegmt=True), 1),
+            (None, 1),
+            ("", 1),
+            ("invalid", 1),
+            ("-5", 1),
+            ("1.5", 1),
+            ("NaN", 1),
+            ("Infinity", 1),
+            pytest.param("9" * 400, 1, id="overflow"),
+        ],
+    )
+    async def test_ai_mode_retry_after(
+        self, respx_mock: respx.MockRouter, value: str | None, expected: int
+    ) -> None:
+        route = respx_mock.get("https://sdk.test/v1/google/ai-mode/search").mock(
+            side_effect=[
+                httpx.Response(503, headers={"Retry-After": value} if value is not None else {}),
+                httpx.Response(200, json={"markdown": "A complete AI Mode answer"}),
+            ]
+        )
+        with (
+            patch("scrapebadger._internal.client.asyncio.sleep") as sleep,
+            patch("scrapebadger._internal.client.time.time", return_value=1_800_000_000.25),
+        ):
+            async with ScrapeBadger(
+                api_key="test_key", base_url="https://sdk.test", max_retries=1
+            ) as client:
+                answer = await client.google.ai_mode.search("Why is the sky blue?")
+
+        assert answer == {"markdown": "A complete AI Mode answer"}
+        assert route.call_count == 2
+        sleep.assert_awaited_once_with(expected)
+
+    @pytest.mark.parametrize("value", ["5", formatdate(1_800_000_005, usegmt=True), "invalid"])
+    def test_ai_mode_from_synchronous_program(
+        self, respx_mock: respx.MockRouter, value: str
+    ) -> None:
+        """The SDK is async-only; synchronous programs enter through asyncio.run()."""
+        route = respx_mock.get("https://sdk.test/v1/google/ai-mode/search").mock(
+            side_effect=[
+                httpx.Response(503, headers={"Retry-After": value}),
+                httpx.Response(200, json={"markdown": "Answer"}),
+            ]
+        )
+
+        async def search() -> str:
+            async with ScrapeBadger(
+                api_key="test_key", base_url="https://sdk.test", max_retries=1
+            ) as client:
+                answer = await client.google.ai_mode.search("Why is the sky blue?")
+                return str(answer["markdown"])
+
+        with (
+            patch("scrapebadger._internal.client.asyncio.sleep") as sleep,
+            patch("scrapebadger._internal.client.time.time", return_value=1_800_000_000),
+        ):
+            assert asyncio.run(search()) == "Answer"
+
+        sleep.assert_awaited_once_with(1 if value == "invalid" else 5)
+        assert route.call_count == 2
+
+    @pytest.mark.parametrize("status", [500, 502, 503, 504])
+    @pytest.mark.parametrize("method", ["get", "get_with_headers", "post"])
+    async def test_shared_retry_path(
+        self, config_one_retry: ClientConfig, method: str, status: int
+    ) -> None:
+        async with BaseClient(config_one_retry) as client:
+            with (
+                patch.object(
+                    client,
+                    "_execute_request",
+                    side_effect=[
+                        httpx.Response(status, headers={"Retry-After": "5"}),
+                        httpx.Response(200, json={"ok": True}),
+                    ],
+                ) as request,
+                patch("scrapebadger._internal.client.asyncio.sleep") as sleep,
+            ):
+                result = await getattr(client, method)("/v1/test")
+        assert (result[0] if method == "get_with_headers" else result) == {"ok": True}
+        assert request.await_count == 2
+        sleep.assert_awaited_once_with(5)
+
+    async def test_retry_limit_and_backoff_are_preserved(self) -> None:
+        async with BaseClient(ClientConfig(api_key="test_key", max_retries=5)) as client:
+            with (
+                patch.object(
+                    client,
+                    "_execute_request",
+                    side_effect=[
+                        httpx.Response(503, headers={"Retry-After": "5"}),
+                        httpx.Response(503, headers={"Retry-After": "invalid"}),
+                        httpx.ConnectTimeout("timeout"),
+                        httpx.Response(503, headers={"Retry-After": "5"}),
+                        httpx.ConnectTimeout("timeout"),
+                        httpx.Response(503, headers={"Retry-After": "5"}),
+                    ],
+                ) as request,
+                patch("scrapebadger._internal.client.asyncio.sleep") as sleep,
+                pytest.raises(ServerError) as error,
+            ):
+                await client.get("/v1/test")
+        assert error.value.status_code == 503
+        assert request.await_count == 6
+        assert sleep.await_args_list == [call(5), call(2), call(4), call(8), call(16)]
+
+    @pytest.mark.parametrize(
+        ("value", "expected"),
+        [("5", 5), ("0", 0), (formatdate(1_800_000_005, usegmt=True), 5), ("invalid", 60)],
+    )
+    async def test_429_keeps_rate_limit_error_without_automatic_retry(
+        self, config_one_retry: ClientConfig, value: str, expected: int
+    ) -> None:
+        async with BaseClient(config_one_retry) as client:
+            with (
+                patch.object(
+                    client,
+                    "_execute_request",
+                    return_value=httpx.Response(429, headers={"Retry-After": value}),
+                ) as request,
+                patch("scrapebadger._internal.client.asyncio.sleep") as sleep,
+                patch("scrapebadger._internal.client.time.time", return_value=1_800_000_000),
+                pytest.raises(RateLimitError) as error,
+            ):
+                await client.get("/v1/test")
+        assert error.value.retry_after == expected
+        assert request.await_count == 1
+        sleep.assert_not_awaited()
+
+
 class TestRetryWarningLogging:
     """Tests that warning logs are emitted on 5xx retries and network errors."""