Skip to content

Commit 0f5fd6e

Browse files
fix(durable): name the callback's operation when a replay registers it
Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 5e3ca8d commit 0f5fd6e

5 files changed

Lines changed: 189 additions & 4 deletions

File tree

‎src/volcano_sdk/_durable_callback_token.py‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from __future__ import annotations
44

5+
import hashlib
56
import inspect
67
from typing import TYPE_CHECKING
78

@@ -48,20 +49,24 @@ def token_submitter(
4849
execution_ref: str,
4950
name: str,
5051
hand_off: Callable[[str], object],
52+
operation_id: str | None = None,
5153
) -> Callable[[str, object], None]:
5254
"""Build the runtime's submitter: register the callback, then hand its token on.
5355
5456
It runs inside the runtime's checkpointed submitter step, so a replay does
5557
not register the callback again, and a step that runs again registers the
56-
same callback and gets the same token.
58+
same callback and gets the same token. `operation_id` names the operation
59+
an earlier invocation opened the callback under; see `replayed_operation`.
5760
5861
Returns:
5962
The submitter the runtime calls with the callback's id.
6063
6164
"""
6265

6366
def submit(callback_id: str, _context: object) -> None:
64-
token = register_callback(api_url, execution_ref, callback_id, name)
67+
token = register_callback(
68+
api_url, execution_ref, callback_id, name, operation_id
69+
)
6570
# A closed callback already timed out or was answered; the wait
6671
# resumes as it did.
6772
if token is not None:
@@ -70,6 +75,41 @@ def submit(callback_id: str, _context: object) -> None:
7075
return submit
7176

7277

78+
def replayed_operation(context: object) -> str | None:
79+
"""Name the operation an earlier invocation opened the next callback under.
80+
81+
Volcano then reads back to that callback rather than only through recent
82+
history. Call it just before the runtime's `wait_for_callback`: the wait
83+
runs in a child context named by the operation about to start, and opens
84+
the callback as that context's first operation. Neither the next operation
85+
nor the checkpoint is public, so a runtime without them names no operation
86+
and the registration reads as a first run's does.
87+
88+
Returns:
89+
The callback's operation id, or None when this invocation opens it.
90+
91+
"""
92+
peek = getattr(context, "_peek_next_operation_id", None)
93+
checkpoint = getattr(getattr(context, "state", None), "get_checkpoint_result", None)
94+
if not callable(peek) or not callable(checkpoint):
95+
return None
96+
operation_id = callback_operation_id(str(peek()))
97+
existent = getattr(checkpoint(operation_id), "is_existent", None)
98+
if not callable(existent) or not existent():
99+
return None
100+
return operation_id
101+
102+
103+
def callback_operation_id(child_id: str) -> str:
104+
"""Name the callback a wait's child context `child_id` opens first.
105+
106+
Returns:
107+
The id the runtime checkpoints the callback under.
108+
109+
"""
110+
return hashlib.blake2b(f"{child_id}-1".encode()).hexdigest()[:64]
111+
112+
73113
def require_synchronous(submitter: object) -> None:
74114
"""Refuse an `async def` submitter before its callback opens.
75115
@@ -99,7 +139,11 @@ def handed_off(result: object) -> None:
99139

100140

101141
def register_callback(
102-
api_url: str, execution_ref: str, callback_id: str, name: str
142+
api_url: str,
143+
execution_ref: str,
144+
callback_id: str,
145+
name: str,
146+
operation_id: str | None = None,
103147
) -> str | None:
104148
"""Ask Volcano for the token that completes the callback.
105149
@@ -118,6 +162,8 @@ def register_callback(
118162
"callback_id": callback_id,
119163
"name": name,
120164
}
165+
if operation_id is not None:
166+
body["operation_id"] = operation_id
121167
try:
122168
answer = post_to_platform(
123169
api_url, PlatformRequest("/durable-callbacks", body, not_ready=_NOT_READY)

‎src/volcano_sdk/_tests/fixtures/durable_callbacks.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,21 @@ class CallbackRegistrations:
1414

1515
def __init__(self) -> None:
1616
self.calls: list[tuple[str, str, str, str]] = []
17+
self.operations: list[str | None] = []
1718
self.closed: bool = False
1819
self._lock: threading.Lock = threading.Lock()
1920

2021
def register(
21-
self, api_url: str, execution_ref: str, callback_id: str, name: str
22+
self,
23+
api_url: str,
24+
execution_ref: str,
25+
callback_id: str,
26+
name: str,
27+
operation_id: str | None = None,
2228
) -> str | None:
2329
with self._lock:
2430
self.calls.append((api_url, execution_ref, callback_id, name))
31+
self.operations.append(operation_id)
2532
return None if self.closed else f"vcb_{callback_id}"
2633

2734
def callback_ids(self) -> list[str]:

‎src/volcano_sdk/_tests/test_durable_authoring.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,13 @@
4444
create_wait_strategy,
4545
)
4646
from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner
47+
from aws_durable_execution_sdk_python_testing.runner import CallbackOperation
4748

4849
from volcano_sdk import ValidationError
50+
from volcano_sdk._durable_callback_token import (
51+
callback_operation_id,
52+
replayed_operation,
53+
)
4954
from volcano_sdk._durable_duration import to_seconds
5055
from volcano_sdk._durable_engine import Engine, load_engine
5156
from volcano_sdk._durable_results import completion_reason
@@ -1843,6 +1848,40 @@ def succeed(runner: DurableFunctionTestRunner, callback: str) -> None:
18431848
assert handoffs.tokens == [f"vcb_{callback}"]
18441849

18451850

1851+
def test_a_replay_names_the_operation_the_first_invocation_opened_the_callback_under(
1852+
callback_registrations: CallbackRegistrations,
1853+
monkeypatch: pytest.MonkeyPatch,
1854+
) -> None:
1855+
named: list[str | None] = []
1856+
1857+
def recorded(context: object) -> str | None:
1858+
operation_id = replayed_operation(context)
1859+
named.append(operation_id)
1860+
return operation_id
1861+
1862+
monkeypatch.setattr("volcano_sdk.durable_authoring.replayed_operation", recorded)
1863+
invocations = Invocations(pay(Handoffs(), timeout="1h"))
1864+
with local_runner(invocations) as runner:
1865+
execution_ref = runner.run_async(input=json.dumps({}))
1866+
invocations.wait_for_one()
1867+
[callback] = callback_registrations.callback_ids()
1868+
runner.send_callback_success(callback, b'"paid"')
1869+
result = runner.wait_for_result(execution_ref, timeout=60)
1870+
1871+
assert result.error is None
1872+
wait = result.get_context("payment")
1873+
[opened] = [
1874+
operation
1875+
for operation in wait.child_operations
1876+
if isinstance(operation, CallbackOperation)
1877+
]
1878+
assert opened.operation_id == callback_operation_id(wait.operation_id)
1879+
# The first invocation opened the callback; the replay that resumed it
1880+
# finds it checkpointed.
1881+
assert named == [None, opened.operation_id]
1882+
assert callback_registrations.operations == [None]
1883+
1884+
18461885
def test_a_callback_failure_keeps_its_type_through_the_replay(
18471886
callback_registrations: CallbackRegistrations,
18481887
) -> None:

‎src/volcano_sdk/_tests/test_durable_callback_token.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import hashlib
45
import inspect
56
import json
7+
from dataclasses import dataclass, field
68
from typing import TYPE_CHECKING, Protocol
79

810
import httpx
@@ -14,6 +16,7 @@
1416
callback_failure,
1517
callback_result,
1618
register_callback,
19+
replayed_operation,
1720
token_submitter,
1821
)
1922
from volcano_sdk._transport_types import decode_json
@@ -151,6 +154,94 @@ def test_the_submitter_hands_the_token_on(platform: InstallPlatform) -> None:
151154
}
152155

153156

157+
def test_a_replayed_submitter_names_the_callbacks_operation(
158+
platform: InstallPlatform,
159+
) -> None:
160+
fake = platform(issued())
161+
handed: list[str] = []
162+
submit = token_submitter(API_URL, "arn:execution", "payment", handed.append, "ab12")
163+
164+
submit("callback-4", None)
165+
166+
assert decode_json(fake.requests[0].content) == {
167+
"execution_ref": "arn:execution",
168+
"callback_id": "callback-4",
169+
"name": "payment",
170+
"operation_id": "ab12",
171+
}
172+
173+
174+
@dataclass(frozen=True)
175+
class Checkpoint:
176+
exists: bool
177+
178+
def is_existent(self) -> bool:
179+
return self.exists
180+
181+
182+
@dataclass
183+
class Checkpoints:
184+
recorded: set[str] = field(default_factory=set[str])
185+
186+
def get_checkpoint_result(self, operation_id: str) -> Checkpoint:
187+
return Checkpoint(operation_id in self.recorded)
188+
189+
190+
@dataclass
191+
class OpaqueCheckpoints:
192+
result: object
193+
194+
def get_checkpoint_result(self, operation_id: str) -> object:
195+
_ = operation_id
196+
return self.result
197+
198+
199+
@dataclass
200+
class RuntimeInternals:
201+
"""Stand in for the runtime's private operation cursor and checkpoint."""
202+
203+
next_operation: str
204+
state: object
205+
206+
def _peek_next_operation_id(self) -> str:
207+
return self.next_operation
208+
209+
210+
@dataclass
211+
class NoCursor:
212+
state: object
213+
214+
215+
def callback_of(child_id: str) -> str:
216+
return hashlib.blake2b(f"{child_id}-1".encode()).hexdigest()[:64]
217+
218+
219+
def test_a_replay_names_the_operation_an_earlier_invocation_opened() -> None:
220+
context = RuntimeInternals("child-7", Checkpoints({callback_of("child-7")}))
221+
222+
assert replayed_operation(context) == callback_of("child-7")
223+
224+
225+
def test_the_invocation_that_opens_the_callback_names_no_operation() -> None:
226+
context = RuntimeInternals("child-7", Checkpoints({callback_of("child-6")}))
227+
228+
assert replayed_operation(context) is None
229+
230+
231+
@pytest.mark.parametrize(
232+
"context",
233+
[
234+
NoCursor(Checkpoints({callback_of("child-7")})),
235+
RuntimeInternals("child-7", None),
236+
RuntimeInternals("child-7", OpaqueCheckpoints(object())),
237+
object(),
238+
],
239+
ids=["no cursor", "no checkpoint", "unreadable checkpoint", "plain object"],
240+
)
241+
def test_a_runtime_without_its_internals_names_no_operation(context: object) -> None:
242+
assert replayed_operation(context) is None
243+
244+
154245
def test_the_submitter_ignores_what_the_hand_off_returns(
155246
platform: InstallPlatform,
156247
) -> None:

‎src/volcano_sdk/durable_authoring.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ def handler(event, ctx):
5151
CallbackFailedError,
5252
callback_failure,
5353
callback_result,
54+
replayed_operation,
5455
require_synchronous,
5556
token_submitter,
5657
)
@@ -551,6 +552,7 @@ def wait_for_callback(
551552
self._context.execution_context.durable_execution_arn,
552553
callback_name,
553554
submitter,
555+
replayed_operation(self._context),
554556
)
555557
try:
556558
result = self._context.wait_for_callback(submit, callback_name, config)

0 commit comments

Comments
 (0)