99#pragma once
1010
1111#include < cstddef>
12+ #include < functional>
1213#include < future>
1314#include < memory>
1415#include < optional>
@@ -28,13 +29,15 @@ struct ET_EXPERIMENTAL ServingRuntimeConfig {
2829 std::size_t max_sessions = 1 ;
2930 // Reported metadata; 0 means unknown.
3031 std::size_t max_context_length = 0 ;
31- // Queued plus executing lifecycle operations. Must be non-zero.
32+ // Queued plus executing lifecycle operations, excluding completion
33+ // callbacks. Must be non-zero.
3234 std::size_t max_pending_operations = 64 ;
3335};
3436
3537// nullopt acknowledges success. Dropping a future does not cancel its
3638// operation.
3739using LifecycleResult ET_EXPERIMENTAL = std::optional<ServingError>;
40+ using LifecycleCallback ET_EXPERIMENTAL = std::function<void (LifecycleResult)>;
3841
3942class ET_EXPERIMENTAL ServingRuntime {
4043 public:
@@ -55,34 +58,45 @@ class ET_EXPERIMENTAL ServingRuntime {
5558 ServingRuntime& operator =(ServingRuntime&&) = delete ;
5659
5760 // Any thread. Keys must be non-empty. Accepted operations run in admission
58- // order, including operations submitted before a previous future completes.
59- // Queue saturation returns CapacityExceeded without changing session state.
61+ // order. Queue saturation reports CapacityExceeded without changing state.
62+ // For all lifecycle operations, callbacks run once on control after
63+ // processing and admission release, or inline on the caller for immediate
64+ // rejection. They run without runtime locks and may race with submission's
65+ // return. Callbacks must do short, nonblocking work: no waits for runtime
66+ // work or runtime shutdown/destruction. Exceptions are logged, not retried.
67+ // An empty callback discards the result. Futures adapt this same completion
68+ // path.
69+ //
6070 // Opens may queue during initialization; success means an empty session is
6171 // owned, or the key was already open. An initial open failure releases its
6272 // slot. Opening an unavailable key fails with NotReady; use reset to retry.
73+ void open_session_async (std::string key, LifecycleCallback on_complete);
6374 std::future<LifecycleResult> open_session_async (std::string key);
6475
6576 // Idempotent, including absent keys. Success releases the logical slot and
6677 // requests Session closure through RAII, not synchronous physical cleanup.
78+ void close_session_async (std::string key, LifecycleCallback on_complete);
6779 std::future<LifecycleResult> close_session_async (std::string key);
6880
6981 // Cold replacement, retaining the key and logical slot throughout. Success
7082 // means reopening completed. Failure destroys the old state and leaves the
7183 // key unavailable, still reserving its slot until reset succeeds or close
7284 // releases it. A missing key returns SessionNotFound. Never uses prefix
7385 // reuse.
86+ void reset_session_async (std::string key, LifecycleCallback on_complete);
7487 std::future<LifecycleResult> reset_session_async (std::string key);
7588
7689 // A synchronized snapshot. Ready means initialized and accepting operations,
7790 // not that capacity is available. Active slots include opening, reopening,
7891 // and unavailable keys, but not commands still waiting to reserve a slot.
7992 ServingInfo info () const ;
8093
81- // Idempotent, including concurrent callers. Stops admission, settles all
82- // accepted futures, and joins the control and Runner threads. No new work is
83- // accepted afterward. Call only from external threads, never engine
84- // callbacks. Destruction calls shutdown; as usual, object lifetime must be
85- // externally synchronized against callers still accessing the runtime.
94+ // Idempotent, including concurrent callers. Stops admission, completes all
95+ // accepted operations, and joins the control and Runner threads. Accepted
96+ // callbacks have returned and their runtime-owned captures are released.
97+ // No new work is accepted afterward. Call only from external threads, never
98+ // engine or lifecycle callbacks. Destruction calls shutdown; as usual, object
99+ // lifetime must be synchronized against callers still accessing the runtime.
86100 void shutdown ();
87101
88102 private:
0 commit comments