Skip to content

Commit 9fd97d0

Browse files
rustyconoverclaude
andcommitted
perf(http): move call state out of the per-turn state token
Adopts the vgi-rpc call/cursor token split. Everything about an invocation that is settled by the time init returns -- the InitRequest, the worker's GlobalInitResponse, and the unwrapped attach plaintext -- moves into a new frozen VgiCallState that the transport seals once and replays, instead of riding the state token on every turn. ScalarExchangeState, TableProducerState and TableInOutExchangeState drop _init_call / _init_response / _plaintext_attach and read them through self._call. What is left is the cursor: _user_state_bytes, and for a scalar exchange nothing at all -- its state token is now empty. VgiCallState is frozen and reachable only through immutable fields because the server hands one instance to every concurrent turn of the stream; that is the contract bind_call_state() documents. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent b6d901c commit 9fd97d0

2 files changed

Lines changed: 130 additions & 87 deletions

File tree

‎vgi/protocol.py‎

Lines changed: 116 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
import logging
3030
import struct
3131
from dataclasses import dataclass, field
32-
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Protocol, get_args, get_origin
32+
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Protocol, cast, get_args, get_origin
3333

3434
import pyarrow as pa
3535
import pyarrow.compute as pc
@@ -726,26 +726,76 @@ class IndexCreateRequest(ArrowSerializableDataclass):
726726
# ---------------------------------------------------------------------------
727727

728728

729+
@dataclass(frozen=True, slots=True, kw_only=True)
730+
class VgiCallState(ArrowSerializableDataclass):
731+
"""The half of a VGI stream's state that is fixed for the whole call.
732+
733+
Every VGI stream — scalar exchange, table producer, table-in-out — is
734+
parameterized by the same trio, and none of it can change once the
735+
stream exists: the init request that opened it, the worker's response to
736+
that init, and the unwrapped attach plaintext.
737+
738+
Under HTTP this travels in its own token, sealed once at ``/init`` and
739+
echoed by the client thereafter (see
740+
:data:`vgi_rpc.metadata.CALL_STATE_KEY`). Keeping it out of the
741+
per-turn state token is worth doing because it dominates: for a plain
742+
``sequence()`` scan it was 7,224 of the 8,344 serialized state bytes,
743+
and re-parsing it cost more than everything else in a turn combined.
744+
745+
Frozen, and reachable only through immutable fields, because the server
746+
hands one instance to every concurrent turn of the stream — the
747+
contract :meth:`StreamState.bind_call_state` sets out.
748+
749+
Attributes:
750+
init_call: The [`InitRequest`][] that opened this stream.
751+
init_response: The worker's [`GlobalInitResponse`][] for that init.
752+
plaintext_attach: Full framework attach plaintext
753+
(``uuid || catalog_bytes``), carried unwrapped so each turn can
754+
shard storage on its UUID without re-opening the auth-scoped
755+
seal; ``None`` outside a catalog context.
756+
757+
"""
758+
759+
init_call: Annotated[InitRequest, ArrowType(pa.binary())]
760+
init_response: Annotated[GlobalInitResponse, ArrowType(pa.binary())]
761+
plaintext_attach: bytes | None = None
762+
763+
764+
@dataclass
765+
class _VgiCallStateHolder:
766+
"""Mixin giving a VGI stream state access to its :class:`VgiCallState`.
767+
768+
The framework calls :meth:`bind_call_state` on both transports — once at
769+
construction under pipe, once per turn under HTTP — so ``self._call``
770+
reads the same on either. It is ``Transient`` because the call state is
771+
the transport's to serialize, not this object's; that is the whole point
772+
of the split.
773+
"""
774+
775+
CALL_STATE_TYPE: ClassVar[type[ArrowSerializableDataclass] | None] = VgiCallState
776+
777+
_call: Annotated[VgiCallState, Transient()] = field(default=None, repr=False) # type: ignore[assignment]
778+
779+
def bind_call_state(self, call_state: ArrowSerializableDataclass | None) -> None:
780+
"""Attach the stream's call state (framework hook)."""
781+
self._call = cast("VgiCallState", call_state)
782+
783+
729784
@dataclass
730-
class ScalarExchangeState(ExchangeState):
785+
class ScalarExchangeState(_VgiCallStateHolder, ExchangeState):
731786
"""Exchange state for scalar function streams.
732787
733788
Calls ``ScalarFunctionGenerator.process()`` per batch. Each ``exchange()``
734789
call sends one input batch and receives one output batch.
735790
736-
``_init_call`` and ``_init_response`` are serialized into the state token
737-
so they survive HTTP round-trips. ``_func_cls`` is transient and restored
738-
via ``rehydrate()``.
791+
The init request, init response, and attach plaintext live in the
792+
stream's :class:`VgiCallState` — sealed once by the transport, not
793+
re-serialized per exchange. This class carries no cursor fields at all:
794+
a scalar exchange is stateless between batches, so its state token is
795+
now empty. ``_func_cls`` is transient and restored via ``rehydrate()``.
739796
740797
"""
741798

742-
_init_call: Annotated[InitRequest, ArrowType(pa.binary())] = field(default=None, repr=False) # type: ignore[assignment]
743-
_init_response: Annotated[GlobalInitResponse, ArrowType(pa.binary())] = field(default=None, repr=False) # type: ignore[assignment]
744-
# Full framework attach plaintext (uuid||catalog_bytes) persisted through
745-
# serialization so each exchange can shard storage on its UUID without
746-
# re-unwrapping (the auth-scoped seal can't be reopened, and ctx.implementation
747-
# is the MetaWorker under subprocess transport).
748-
_plaintext_attach: bytes | None = field(default=None, repr=False)
749799
_func_cls: Annotated[type[ScalarFunctionGenerator], Transient()] = field(default=None, repr=False) # type: ignore[assignment]
750800
_vgi_tracer: Annotated[VgiTracer, Transient()] = field(default_factory=get_noop_tracer, repr=False)
751801

@@ -754,7 +804,7 @@ def rehydrate(self, implementation: object) -> None:
754804
from vgi.worker import Worker
755805

756806
worker: Worker = implementation # type: ignore[assignment]
757-
self._func_cls = worker._resolve_function(self._init_call.bind_call) # type: ignore[assignment]
807+
self._func_cls = worker._resolve_function(self._call.init_call.bind_call) # type: ignore[assignment]
758808
self._vgi_tracer = worker._vgi_tracer
759809

760810
def exchange(self, input: AnnotatedBatch, out: OutputCollector, ctx: CallContext) -> None:
@@ -775,20 +825,20 @@ def exchange(self, input: AnnotatedBatch, out: OutputCollector, ctx: CallContext
775825
timer = _timed_exchange(
776826
self._vgi_tracer,
777827
"vgi.execute.scalar",
778-
self._init_call.bind_call.function_name,
779-
self._init_call.bind_call.function_type.value,
780-
self._init_response.execution_id,
828+
self._call.init_call.bind_call.function_name,
829+
self._call.init_call.bind_call.function_type.value,
830+
self._call.init_response.execution_id,
781831
)
782832
with timer:
783833
output = cls.process(
784834
batch=batch,
785-
init_call=self._init_call,
786-
init_response=self._init_response,
835+
init_call=self._call.init_call,
836+
init_response=self._call.init_response,
787837
# Shard on the UUID of the full attach plaintext persisted at init.
788838
storage=BoundStorage(
789839
cls.storage,
790-
self._init_response.execution_id,
791-
attach_plaintext=self._plaintext_attach,
840+
self._call.init_response.execution_id,
841+
attach_plaintext=self._call.plaintext_attach,
792842
),
793843
auth_context=ctx.auth,
794844
)
@@ -1301,7 +1351,7 @@ def __getattr__(self, name: str) -> Any:
13011351

13021352

13031353
@dataclass
1304-
class TableProducerState(ProducerState):
1354+
class TableProducerState(_VgiCallStateHolder, ProducerState):
13051355
"""Producer state for table function streams.
13061356
13071357
Calls ``TableFunctionGenerator.process()`` per tick. Each ``produce()``
@@ -1311,24 +1361,18 @@ class TableProducerState(ProducerState):
13111361
filters from the init request are automatically applied to each output
13121362
batch after ``process()`` produces it.
13131363
1314-
``_init_call`` and ``_init_response`` are serialized into the state token
1315-
so they survive HTTP round-trips. Transient fields are restored via
1316-
``rehydrate()``.
1364+
The init request, init response, and attach plaintext live in the
1365+
stream's :class:`VgiCallState`, which the transport seals once at init.
1366+
What remains here is the cursor: ``_user_state_bytes``, and nothing
1367+
else. Transient fields are restored via ``rehydrate()``.
13171368
13181369
``_user_state`` is serialized when it is an ``ArrowSerializableDataclass``,
13191370
allowing iteration state to survive HTTP round-trips. When the state type
13201371
is not serializable, it falls back to ``initial_state()`` on rehydration.
13211372
13221373
"""
13231374

1324-
_init_call: Annotated[InitRequest, ArrowType(pa.binary())] = field(default=None, repr=False) # type: ignore[assignment]
1325-
_init_response: Annotated[GlobalInitResponse, ArrowType(pa.binary())] = field(default=None, repr=False) # type: ignore[assignment]
13261375
_user_state_bytes: bytes | None = field(default=None, repr=False)
1327-
# Plaintext attach for bodies that read it as user data. ``_init_call`` now
1328-
# carries the SEALED attach (storage shards on it via request=); this carries
1329-
# the unwrapped form through serialization so rehydrate can set
1330-
# params.attach_opaque_data without re-unwrapping (the seal is auth-scoped).
1331-
_plaintext_attach: bytes | None = field(default=None, repr=False)
13321376
_func_cls: Annotated[type[TableFunctionGenerator[Any]], Transient()] = field(default=None, repr=False) # type: ignore[assignment]
13331377
_params: Annotated[ProcessParams[Any], Transient()] = field(default=None, repr=False) # type: ignore[arg-type]
13341378
_user_state: Annotated[Any, Transient()] = field(default=None, repr=False)
@@ -1362,30 +1406,30 @@ def rehydrate(self, implementation: object) -> None:
13621406
from vgi.worker import Worker
13631407

13641408
worker: Worker = implementation # type: ignore[assignment]
1365-
func_cls = worker._resolve_function(self._init_call.bind_call)
1409+
func_cls = worker._resolve_function(self._call.init_call.bind_call)
13661410
assert issubclass(func_cls, TableFunctionGenerator)
13671411
self._func_cls = func_cls
13681412
self._vgi_tracer = worker._vgi_tracer
1369-
proj_ids = _effective_projection_ids(func_cls, self._init_call.projection_ids)
1370-
output_schema = project_schema(proj_ids, self._init_call.output_schema)
1413+
proj_ids = _effective_projection_ids(func_cls, self._call.init_call.projection_ids)
1414+
output_schema = project_schema(proj_ids, self._call.init_call.output_schema)
13711415
self._params = ProcessParams(
13721416
args=func_cls._parse_arguments(
1373-
func_cls.FunctionArguments, self._init_call.bind_call.arguments, blended=func_cls._is_blended()
1417+
func_cls.FunctionArguments, self._call.init_call.bind_call.arguments, blended=func_cls._is_blended()
13741418
),
1375-
init_call=self._init_call,
1376-
init_response=self._init_response,
1419+
init_call=self._call.init_call,
1420+
init_response=self._call.init_response,
13771421
output_schema=output_schema,
1378-
settings=_batch_to_scalar_dict(self._init_call.bind_call.settings),
1379-
secrets=SecretsAccessor(self._init_call.bind_call.secrets).to_dict(),
1422+
settings=_batch_to_scalar_dict(self._call.init_call.bind_call.settings),
1423+
secrets=SecretsAccessor(self._call.init_call.bind_call.secrets).to_dict(),
13801424
# Rehydrated tick: the auth-scoped seal can't be reopened here, so we
13811425
# shard storage on the full plaintext (uuid||catalog_bytes) the init
13821426
# state persisted; the body sees only the stripped catalog bytes.
13831427
storage=BoundStorage(
13841428
func_cls.storage,
1385-
self._init_response.execution_id,
1386-
attach_plaintext=self._plaintext_attach,
1429+
self._call.init_response.execution_id,
1430+
attach_plaintext=self._call.plaintext_attach,
13871431
),
1388-
attach_opaque_data=attach_catalog_bytes(self._plaintext_attach),
1432+
attach_opaque_data=attach_catalog_bytes(self._call.plaintext_attach),
13891433
)
13901434
# Restore _user_state from serialized bytes if available
13911435
if self._user_state_bytes is not None:
@@ -1401,10 +1445,10 @@ def rehydrate(self, implementation: object) -> None:
14011445
# Re-derive pushdown filters (triggers same logic as __post_init__)
14021446
if func_cls._should_auto_apply_filters():
14031447
self._auto_apply = True
1404-
if self._init_call.pushdown_filters is not None:
1448+
if self._call.init_call.pushdown_filters is not None:
14051449
self._pushdown_filters = func_cls.pushdown_filters(
1406-
self._init_call.pushdown_filters,
1407-
join_keys=self._init_call.join_keys,
1450+
self._call.init_call.pushdown_filters,
1451+
join_keys=self._call.init_call.join_keys,
14081452
)
14091453

14101454
def process(self, input: AnnotatedBatch, out: OutputCollector, ctx: CallContext) -> None:
@@ -1451,15 +1495,15 @@ def produce(self, out: OutputCollector, ctx: CallContext) -> None:
14511495
timer = _timed_exchange(
14521496
self._vgi_tracer,
14531497
"vgi.execute.table",
1454-
self._init_call.bind_call.function_name,
1455-
self._init_call.bind_call.function_type.value,
1456-
self._init_response.execution_id,
1498+
self._call.init_call.bind_call.function_name,
1499+
self._call.init_call.bind_call.function_type.value,
1500+
self._call.init_response.execution_id,
14571501
)
14581502
with timer:
14591503
tracking_out = _TrackingOutputCollector(
14601504
out,
14611505
supports_batch_index=self._func_cls._supports_batch_index(),
1462-
partition_fields=_partition_fields_from_schema(self._init_call.output_schema),
1506+
partition_fields=_partition_fields_from_schema(self._call.init_call.output_schema),
14631507
partition_kind=self._func_cls._partition_kind(),
14641508
)
14651509
if self._auto_apply and self._pushdown_filters is not None:
@@ -1485,7 +1529,7 @@ def on_cancel(self, ctx: CallContext) -> None:
14851529

14861530

14871531
@dataclass
1488-
class TableInOutExchangeState(ExchangeState):
1532+
class TableInOutExchangeState(_VgiCallStateHolder, ExchangeState):
14891533
"""Exchange state for table-in-out function streams (INPUT phase).
14901534
14911535
Calls ``TableInOutGenerator.process()`` per input batch. Each
@@ -1494,23 +1538,17 @@ class TableInOutExchangeState(ExchangeState):
14941538
When ``auto_apply_filters`` is enabled, pushdown filters from the init
14951539
request are automatically applied to each output batch.
14961540
1497-
``_init_call`` and ``_init_response`` are serialized into the state token
1498-
so they survive HTTP round-trips. Transient fields are restored via
1499-
``rehydrate()``.
1541+
The init request, init response, and attach plaintext live in the
1542+
stream's :class:`VgiCallState`, which the transport seals once at init.
1543+
What remains here is the cursor: ``_user_state_bytes``, and nothing
1544+
else. Transient fields are restored via ``rehydrate()``.
15001545
15011546
``_user_state`` is serialized when it is an ``ArrowSerializableDataclass``,
15021547
allowing iteration state to survive HTTP round-trips.
15031548
15041549
"""
15051550

1506-
_init_call: Annotated[InitRequest, ArrowType(pa.binary())] = field(default=None, repr=False) # type: ignore[assignment]
1507-
_init_response: Annotated[GlobalInitResponse, ArrowType(pa.binary())] = field(default=None, repr=False) # type: ignore[assignment]
15081551
_user_state_bytes: bytes | None = field(default=None, repr=False)
1509-
# Plaintext attach for bodies that read it as user data. ``_init_call`` now
1510-
# carries the SEALED attach (storage shards on it via request=); this carries
1511-
# the unwrapped form through serialization so rehydrate can set
1512-
# params.attach_opaque_data without re-unwrapping (the seal is auth-scoped).
1513-
_plaintext_attach: bytes | None = field(default=None, repr=False)
15141552
_func_cls: Annotated[type[TableInOutGenerator[Any]], Transient()] = field(default=None, repr=False) # type: ignore[assignment]
15151553
_params: Annotated[ProcessParams[Any], Transient()] = field(default=None, repr=False) # type: ignore[arg-type]
15161554
_user_state: Annotated[Any, Transient()] = field(default=None, repr=False)
@@ -1540,30 +1578,30 @@ def rehydrate(self, implementation: object) -> None:
15401578
from vgi.worker import Worker
15411579

15421580
worker: Worker = implementation # type: ignore[assignment]
1543-
func_cls = worker._resolve_function(self._init_call.bind_call)
1581+
func_cls = worker._resolve_function(self._call.init_call.bind_call)
15441582
assert issubclass(func_cls, TableInOutGenerator)
15451583
self._func_cls = func_cls
15461584
self._vgi_tracer = worker._vgi_tracer
1547-
proj_ids = _effective_projection_ids(func_cls, self._init_call.projection_ids)
1548-
output_schema = project_schema(proj_ids, self._init_call.output_schema)
1585+
proj_ids = _effective_projection_ids(func_cls, self._call.init_call.projection_ids)
1586+
output_schema = project_schema(proj_ids, self._call.init_call.output_schema)
15491587
self._params = ProcessParams(
15501588
args=func_cls._parse_arguments(
1551-
func_cls.FunctionArguments, self._init_call.bind_call.arguments, blended=func_cls._is_blended()
1589+
func_cls.FunctionArguments, self._call.init_call.bind_call.arguments, blended=func_cls._is_blended()
15521590
),
1553-
init_call=self._init_call,
1554-
init_response=self._init_response,
1591+
init_call=self._call.init_call,
1592+
init_response=self._call.init_response,
15551593
output_schema=output_schema,
1556-
settings=_batch_to_scalar_dict(self._init_call.bind_call.settings),
1557-
secrets=SecretsAccessor(self._init_call.bind_call.secrets).to_dict(),
1594+
settings=_batch_to_scalar_dict(self._call.init_call.bind_call.settings),
1595+
secrets=SecretsAccessor(self._call.init_call.bind_call.secrets).to_dict(),
15581596
# Rehydrated tick: shard storage on the full plaintext the init state
15591597
# persisted (the auth-scoped seal can't be reopened here); the body
15601598
# sees only the stripped catalog bytes.
15611599
storage=BoundStorage(
15621600
func_cls.storage,
1563-
self._init_response.execution_id,
1564-
attach_plaintext=self._plaintext_attach,
1601+
self._call.init_response.execution_id,
1602+
attach_plaintext=self._call.plaintext_attach,
15651603
),
1566-
attach_opaque_data=attach_catalog_bytes(self._plaintext_attach),
1604+
attach_opaque_data=attach_catalog_bytes(self._call.plaintext_attach),
15671605
)
15681606
# Restore _user_state from serialized bytes if available
15691607
if self._user_state_bytes is not None:
@@ -1576,10 +1614,10 @@ def rehydrate(self, implementation: object) -> None:
15761614
self._user_state = func_cls.initial_state(self._params)
15771615
if func_cls._should_auto_apply_filters():
15781616
self._auto_apply = True
1579-
if self._init_call.pushdown_filters is not None:
1617+
if self._call.init_call.pushdown_filters is not None:
15801618
self._pushdown_filters = func_cls.pushdown_filters(
1581-
self._init_call.pushdown_filters,
1582-
join_keys=self._init_call.join_keys,
1619+
self._call.init_call.pushdown_filters,
1620+
join_keys=self._call.init_call.join_keys,
15831621
)
15841622

15851623
def exchange(self, input: AnnotatedBatch, out: OutputCollector, ctx: CallContext) -> None:
@@ -1602,15 +1640,15 @@ def exchange(self, input: AnnotatedBatch, out: OutputCollector, ctx: CallContext
16021640
timer = _timed_exchange(
16031641
self._vgi_tracer,
16041642
"vgi.execute.table_in_out",
1605-
self._init_call.bind_call.function_name,
1606-
self._init_call.bind_call.function_type.value,
1607-
self._init_response.execution_id,
1643+
self._call.init_call.bind_call.function_name,
1644+
self._call.init_call.bind_call.function_type.value,
1645+
self._call.init_response.execution_id,
16081646
)
16091647
with timer:
16101648
tracking_out = _TrackingOutputCollector(
16111649
out,
16121650
supports_batch_index=self._func_cls._supports_batch_index(),
1613-
partition_fields=_partition_fields_from_schema(self._init_call.output_schema),
1651+
partition_fields=_partition_fields_from_schema(self._call.init_call.output_schema),
16141652
partition_kind=self._func_cls._partition_kind(),
16151653
)
16161654
if self._auto_apply and self._pushdown_filters is not None:

0 commit comments

Comments
 (0)