Skip to content

Commit 3e30e6b

Browse files
lesnik512claude
andcommitted
feat(resilience): add AsyncCircuitBreaker (classic consecutive-failure breaker)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 1e9a2c3 commit 3e30e6b

5 files changed

Lines changed: 599 additions & 3 deletions

File tree

‎src/httpware/__init__.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,21 @@
3838
before_request,
3939
on_error,
4040
)
41-
from httpware.middleware.resilience import AsyncBulkhead, AsyncRetry, AsyncTimeout, Bulkhead, Retry, RetryBudget
41+
from httpware.middleware.resilience import (
42+
AsyncBulkhead,
43+
AsyncCircuitBreaker,
44+
AsyncRetry,
45+
AsyncTimeout,
46+
Bulkhead,
47+
Retry,
48+
RetryBudget,
49+
)
4250

4351

4452
__all__ = [
4553
"STATUS_TO_EXCEPTION",
4654
"AsyncBulkhead",
55+
"AsyncCircuitBreaker",
4756
"AsyncClient",
4857
"AsyncMiddleware",
4958
"AsyncNext",
Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
"""Resilience primitives: Bulkhead/AsyncBulkhead, Retry/AsyncRetry, RetryBudget, AsyncTimeout."""
1+
"""Resilience primitives: Bulkhead, AsyncBulkhead, AsyncCircuitBreaker, Retry, AsyncRetry, RetryBudget, AsyncTimeout."""
22

33
from httpware.middleware.resilience.budget import RetryBudget
44
from httpware.middleware.resilience.bulkhead import AsyncBulkhead, Bulkhead
5+
from httpware.middleware.resilience.circuit_breaker import AsyncCircuitBreaker
56
from httpware.middleware.resilience.retry import AsyncRetry, Retry
67
from httpware.middleware.resilience.timeout import AsyncTimeout
78

89

9-
__all__ = ["AsyncBulkhead", "AsyncRetry", "AsyncTimeout", "Bulkhead", "Retry", "RetryBudget"]
10+
__all__ = ["AsyncBulkhead", "AsyncCircuitBreaker", "AsyncRetry", "AsyncTimeout", "Bulkhead", "Retry", "RetryBudget"]
Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
"""CircuitBreaker + AsyncCircuitBreaker — classic consecutive-failure circuit breaker.
2+
3+
See planning/specs/2026-06-13-circuit-breaker-and-timeout-design.md for the contract.
4+
5+
A counted failure is a NetworkError, an httpware TimeoutError, or a StatusError whose
6+
status_code is in the effective failure set (default: all 5xx). 4xx — including 429 —
7+
count as successes: 429 means healthy-but-throttling, and tripping on it amplifies
8+
incidents. Any other exception propagates without affecting circuit state.
9+
10+
State machine (classic / consecutive-failure):
11+
CLOSED — forward; count consecutive counted-failures; open at failure_threshold.
12+
OPEN — fast-fail with CircuitOpenError; after reset_timeout the next request
13+
becomes the half-open probe.
14+
HALF_OPEN — admit exactly one probe at a time; success_threshold consecutive probe
15+
successes close the circuit; one probe failure re-opens it.
16+
17+
The lock-free _CircuitBreakerState holds the transition logic, shared by both wrappers.
18+
AsyncCircuitBreaker relies on asyncio atomicity (no await inside a transition) plus a
19+
single-event-loop guard; CircuitBreaker (sync) serializes transitions with a
20+
threading.Lock. Both are sharable across clients (one shared circuit); a sync instance
21+
cannot be shared with an async one.
22+
"""
23+
24+
import asyncio
25+
import enum
26+
import logging
27+
import threading
28+
import time
29+
import typing
30+
from collections.abc import Callable
31+
32+
import httpx2
33+
34+
from httpware._internal.observability import _emit_event
35+
from httpware.errors import CircuitOpenError, NetworkError, StatusError, TimeoutError # noqa: A004
36+
from httpware.middleware import AsyncNext
37+
38+
39+
_FAILURE_THRESHOLD_INVALID = "failure_threshold must be >= 1"
40+
_RESET_TIMEOUT_INVALID = "reset_timeout must be >= 0"
41+
_SUCCESS_THRESHOLD_INVALID = "success_threshold must be >= 1"
42+
_CROSS_LOOP_MSG = (
43+
"AsyncCircuitBreaker is bound to a single event loop. First seen on {first!r}; "
44+
"current request is on {current!r}. Use one AsyncCircuitBreaker per loop; "
45+
"cross-thread sharing requires the sync CircuitBreaker primitive."
46+
)
47+
48+
_DEFAULT_FAILURE_STATUS_CODES = frozenset(range(500, 600))
49+
50+
_ROLE_CLOSED = "closed"
51+
_ROLE_PROBE = "probe"
52+
53+
_LOGGER = logging.getLogger("httpware.circuit_breaker")
54+
55+
56+
class _CircuitState(enum.Enum):
57+
CLOSED = "closed"
58+
OPEN = "open"
59+
HALF_OPEN = "half_open"
60+
61+
62+
class _CircuitBreakerState:
63+
"""Lock-free circuit-breaker state machine shared by the sync + async wrappers.
64+
65+
Every method is synchronous and performs no I/O beyond logging. The async wrapper
66+
calls these directly (atomic under a single event loop because no await occurs
67+
inside a transition); the sync wrapper wraps each call in a threading.Lock.
68+
"""
69+
70+
def __init__(
71+
self,
72+
*,
73+
failure_threshold: int,
74+
reset_timeout: float,
75+
success_threshold: int,
76+
failure_status_codes: frozenset[int] | None,
77+
now: Callable[[], float],
78+
) -> None:
79+
if failure_threshold < 1:
80+
raise ValueError(_FAILURE_THRESHOLD_INVALID)
81+
if reset_timeout < 0:
82+
raise ValueError(_RESET_TIMEOUT_INVALID)
83+
if success_threshold < 1:
84+
raise ValueError(_SUCCESS_THRESHOLD_INVALID)
85+
self._failure_threshold = failure_threshold
86+
self._reset_timeout = reset_timeout
87+
self._success_threshold = success_threshold
88+
self._failure_status_codes = (
89+
failure_status_codes if failure_status_codes is not None else _DEFAULT_FAILURE_STATUS_CODES
90+
)
91+
self._now = now
92+
self._state = _CircuitState.CLOSED
93+
self._consecutive_failures = 0
94+
self._consecutive_successes = 0
95+
self._opened_at = 0.0
96+
self._probe_in_flight = False
97+
98+
def is_failure_status(self, status_code: int) -> bool:
99+
return status_code in self._failure_status_codes
100+
101+
def admit(self, request: httpx2.Request) -> str:
102+
"""Decide the request's role, or raise CircuitOpenError. No await inside."""
103+
if self._state is _CircuitState.CLOSED:
104+
return _ROLE_CLOSED
105+
if self._state is _CircuitState.OPEN:
106+
elapsed = self._now() - self._opened_at
107+
if elapsed >= self._reset_timeout:
108+
self._state = _CircuitState.HALF_OPEN
109+
self._probe_in_flight = True
110+
self._emit(request, "circuit.half_open", logging.INFO, "circuit half-open — admitting probe", {})
111+
return _ROLE_PROBE
112+
retry_after = max(0.0, self._reset_timeout - elapsed)
113+
self._emit(
114+
request,
115+
"circuit.rejected",
116+
logging.WARNING,
117+
"circuit open — rejecting request",
118+
{"retry_after": retry_after},
119+
)
120+
raise CircuitOpenError(retry_after=retry_after)
121+
# HALF_OPEN
122+
if self._probe_in_flight:
123+
self._emit(
124+
request,
125+
"circuit.rejected",
126+
logging.WARNING,
127+
"circuit half-open — rejecting request (probe in flight)",
128+
{"retry_after": None},
129+
)
130+
raise CircuitOpenError(retry_after=None)
131+
self._probe_in_flight = True
132+
return _ROLE_PROBE
133+
134+
def on_success(self, role: str, request: httpx2.Request) -> None:
135+
if role == _ROLE_PROBE:
136+
self._probe_in_flight = False
137+
if self._state is _CircuitState.CLOSED:
138+
self._consecutive_failures = 0
139+
elif self._state is _CircuitState.HALF_OPEN:
140+
self._consecutive_successes += 1
141+
if self._consecutive_successes >= self._success_threshold:
142+
self._state = _CircuitState.CLOSED
143+
self._consecutive_failures = 0
144+
self._consecutive_successes = 0
145+
self._emit(request, "circuit.closed", logging.INFO, "circuit closed — service recovered", {})
146+
147+
def on_failure(self, role: str, request: httpx2.Request) -> None:
148+
if role == _ROLE_PROBE:
149+
self._probe_in_flight = False
150+
if self._state is _CircuitState.CLOSED:
151+
self._consecutive_failures += 1
152+
if self._consecutive_failures >= self._failure_threshold:
153+
self._open(request, failures=self._consecutive_failures)
154+
elif self._state is _CircuitState.HALF_OPEN:
155+
self._consecutive_successes = 0
156+
self._open(request, failures=1)
157+
158+
def release_probe(self, role: str) -> None:
159+
"""Release the probe slot without recording success or failure (non-counted exc)."""
160+
if role == _ROLE_PROBE:
161+
self._probe_in_flight = False
162+
163+
def _open(self, request: httpx2.Request, *, failures: int) -> None:
164+
self._state = _CircuitState.OPEN
165+
self._opened_at = self._now()
166+
self._emit(
167+
request,
168+
"circuit.opened",
169+
logging.WARNING,
170+
"circuit opened — failure threshold reached",
171+
{"failure_threshold": self._failure_threshold, "failures": failures},
172+
)
173+
174+
def _emit(
175+
self,
176+
request: httpx2.Request,
177+
event_name: str,
178+
level: int,
179+
message: str,
180+
attributes: dict[str, typing.Any],
181+
) -> None:
182+
_emit_event(
183+
_LOGGER,
184+
event_name,
185+
level=level,
186+
message=message,
187+
attributes={**attributes, "method": request.method, "url": str(request.url)},
188+
)
189+
190+
191+
class AsyncCircuitBreaker:
192+
"""Async classic circuit breaker middleware. See the module docstring for the contract."""
193+
194+
def __init__(
195+
self,
196+
*,
197+
failure_threshold: int = 5,
198+
reset_timeout: float = 30.0,
199+
success_threshold: int = 1,
200+
failure_status_codes: frozenset[int] | None = None,
201+
_now: Callable[[], float] = time.monotonic,
202+
) -> None:
203+
self._state = _CircuitBreakerState(
204+
failure_threshold=failure_threshold,
205+
reset_timeout=reset_timeout,
206+
success_threshold=success_threshold,
207+
failure_status_codes=failure_status_codes,
208+
now=_now,
209+
)
210+
self._loop: asyncio.AbstractEventLoop | None = None
211+
self._loop_lock = threading.Lock()
212+
213+
def _check_loop(self) -> None:
214+
current = asyncio.get_running_loop()
215+
cached = self._loop
216+
if cached is current:
217+
return
218+
if cached is not None:
219+
raise RuntimeError(_CROSS_LOOP_MSG.format(first=cached, current=current))
220+
with self._loop_lock:
221+
if self._loop is None:
222+
self._loop = current
223+
# pragma below: inner double-check-with-lock race arm; only reachable when
224+
# two threads simultaneously pass the outer check, which single-threaded
225+
# tests can't trigger.
226+
elif self._loop is not current: # pragma: no cover
227+
raise RuntimeError(_CROSS_LOOP_MSG.format(first=self._loop, current=current))
228+
229+
async def __call__(self, request: httpx2.Request, next: AsyncNext) -> httpx2.Response: # noqa: A002
230+
"""Admit, forward, then record the outcome. Fast-fail when the circuit is not closed."""
231+
self._check_loop()
232+
role = self._state.admit(request)
233+
try:
234+
response = await next(request)
235+
except StatusError as exc:
236+
if self._state.is_failure_status(exc.response.status_code):
237+
self._state.on_failure(role, request)
238+
else:
239+
self._state.on_success(role, request)
240+
raise
241+
except (NetworkError, TimeoutError):
242+
self._state.on_failure(role, request)
243+
raise
244+
except BaseException:
245+
self._state.release_probe(role)
246+
raise
247+
self._state.on_success(role, request)
248+
return response

0 commit comments

Comments
 (0)