6363 TableInfo ,
6464 ViewInfo ,
6565)
66+ from vgi .filter_v2 import delta_revisions , replaying_accepted_state
6667from vgi .function import StreamStateCodec , _is_state_codec
6768from vgi .function_storage import BoundStorage , FrameworkNS , attach_catalog_bytes
6869from vgi .invocation import BindResponse , FunctionType , GlobalInitResponse
@@ -1827,17 +1828,84 @@ def __getattr__(self, name: str) -> Any:
18271828 return getattr (self ._inner , name )
18281829
18291830
1830- def _apply_filter_delta_bytes (current : Any , filter_bytes : bytes ) -> Any :
1831- """Decode one IPC delta and apply it to existing immutable filter state."""
1832- if current is None :
1833- raise ValueError ("dynamic filter delta received without an initial snapshot" )
1831+ def _decode_filter_delta (filter_bytes : bytes ) -> pa .RecordBatch :
1832+ """Decode one IPC dynamic-filter delta into its single document batch."""
18341833 if len (filter_bytes ) > (17 << 20 ):
18351834 raise ValueError ("dynamic filter IPC payload exceeds the encoded-size limit" )
18361835 table = pa .ipc .open_stream (filter_bytes ).read_all ()
18371836 batches = table .to_batches ()
18381837 if len (batches ) != 1 :
18391838 raise ValueError ("dynamic filter metadata must contain exactly one RecordBatch" )
1840- return current .apply_delta (batches [0 ])
1839+ return batches [0 ]
1840+
1841+
1842+ def _apply_filter_delta_bytes (current : Any , filter_bytes : bytes ) -> Any :
1843+ """Decode one IPC delta and apply it to existing immutable filter state."""
1844+ if current is None :
1845+ raise ValueError ("dynamic filter delta received without an initial snapshot" )
1846+ return current .apply_delta (_decode_filter_delta (filter_bytes ))
1847+
1848+
1849+ def _record_filter_delta (current : Any , history : list [bytes ], filter_bytes : bytes ) -> tuple [Any , list [bytes ], list [str ]]:
1850+ """Apply one tick's dynamic-filter delta; return the new filters and what a cursor must carry.
1851+
1852+ An HTTP stream cannot keep parsed filters between turns, so its cursor
1853+ carries the deltas it has applied and :func:`_replay_filter_history` rebuilds
1854+ the state from them on the next turn. Carrying *every* delta made each turn
1855+ replay all of its predecessors -- quadratic in the tick count, and a Top-N
1856+ scan gets a delta on nearly every tick (``dynamic_filter.test`` took ~200 s).
1857+
1858+ So the history is compacted to the deltas that installed some predicate's
1859+ *current* revision, tombstones included: for each ``(id, revision)`` of the
1860+ live state, the first delta carrying it. Replaying just those reproduces the
1861+ same predicates, values and revisions -- earlier updates to an ID are
1862+ overwritten by its current revision, later ones were stale and stay stale --
1863+ and the history stays bounded by the number of predicate IDs instead of
1864+ growing per tick. Replay cannot always reproduce predicate *order* (an ID
1865+ removed and later re-added moves to the end), so the order is returned
1866+ alongside to be restored explicitly.
1867+
1868+ Args:
1869+ current: The stream's current ``PushdownFilters``.
1870+ history: The compacted deltas applied so far.
1871+ filter_bytes: The IPC bytes of the delta this tick carried.
1872+
1873+ Returns:
1874+ ``(filters, history, order)``: the updated filters, the compacted
1875+ history including this delta if it is still needed, and the live
1876+ predicate IDs in order.
1877+
1878+ """
1879+ updated = _apply_filter_delta_bytes (current , filter_bytes )
1880+ wanted = set (updated ._revision_map ().items ())
1881+ kept : list [bytes ] = []
1882+ for delta in [* history , filter_bytes ]:
1883+ carried = set (delta_revisions (_decode_filter_delta (delta ))) & wanted
1884+ if carried :
1885+ kept .append (delta )
1886+ wanted -= carried
1887+ return updated , kept , [p .id for p in updated .predicates ]
1888+
1889+
1890+ def _replay_filter_history (filters : Any , history : list [bytes ], order : list [str ]) -> Any :
1891+ """Rebuild a turn's filters from the init snapshot plus the carried deltas.
1892+
1893+ Runs inside :func:`vgi.filter_v2.replaying_accepted_state` (see the call
1894+ sites): every document here was validated when it first arrived, and both
1895+ tokens are sealed by the worker.
1896+
1897+ Args:
1898+ filters: The ``PushdownFilters`` parsed from the init snapshot.
1899+ history: The compacted deltas from :func:`_record_filter_delta`.
1900+ order: The live predicate order recorded alongside them.
1901+
1902+ Returns:
1903+ The stream's current ``PushdownFilters``.
1904+
1905+ """
1906+ for filter_bytes in history :
1907+ filters = _apply_filter_delta_bytes (filters , filter_bytes )
1908+ return filters ._with_predicate_order (order ) if history else filters
18411909
18421910
18431911@dataclass
@@ -1868,7 +1936,10 @@ class TableProducerState(_VgiCallStateHolder, ProducerState):
18681936 _user_state : Annotated [Any , Transient ()] = field (default = None , repr = False )
18691937 _pushdown_filters : Annotated [Any , Transient ()] = field (default = None , repr = False ) # PushdownFilters | None
18701938 _auto_apply : Annotated [bool , Transient ()] = field (default = False , repr = False )
1939+ # The dynamic-filter deltas this stream must replay on an HTTP turn, compacted by
1940+ # _record_filter_delta, and the live predicate order they rebuild.
18711941 _filter_delta_history : list [bytes ] = field (default_factory = list , repr = False )
1942+ _filter_predicate_order : list [str ] = field (default_factory = list , repr = False )
18721943 _vgi_tracer : Annotated [VgiTracer , Transient ()] = field (default_factory = get_noop_tracer , repr = False )
18731944 # Conditional-revalidation validators read off the first tick's custom_metadata
18741945 # and surfaced to the generator via ProcessParams (M6). None on a normal call.
@@ -1935,13 +2006,17 @@ def rehydrate(self, implementation: object) -> None:
19352006 # manual consumers see the same required/current state after HTTP rehydrate.
19362007 self ._auto_apply = func_cls ._should_auto_apply_filters ()
19372008 if self ._call .init_call .pushdown_filters is not None :
1938- self ._pushdown_filters = func_cls .pushdown_filters (
1939- self ._call .init_call .pushdown_filters ,
1940- join_keys = self ._call .init_call .join_keys ,
1941- output_schema = self ._call .init_call .output_schema ,
1942- )
1943- for filter_bytes in self ._filter_delta_history :
1944- self ._pushdown_filters = _apply_filter_delta_bytes (self ._pushdown_filters , filter_bytes )
2009+ # Everything re-parsed here was validated when it first arrived.
2010+ with replaying_accepted_state ():
2011+ self ._pushdown_filters = _replay_filter_history (
2012+ func_cls .pushdown_filters (
2013+ self ._call .init_call .pushdown_filters ,
2014+ join_keys = self ._call .init_call .join_keys ,
2015+ output_schema = self ._call .init_call .output_schema ,
2016+ ),
2017+ self ._filter_delta_history ,
2018+ self ._filter_predicate_order ,
2019+ )
19452020 self ._params = dataclasses .replace (self ._params , current_pushdown_filters = self ._pushdown_filters )
19462021 # Restore _user_state from serialized bytes if available
19472022 if self ._user_state_bytes is not None :
@@ -1978,8 +2053,9 @@ def _update_filters_from_metadata(self, encoded_filters: bytes) -> None:
19782053 if len (encoded_filters ) > (24 << 20 ):
19792054 raise ValueError ("base64 dynamic filter metadata exceeds the encoded-size limit" )
19802055 filter_bytes = base64 .b64decode (encoded_filters , validate = True )
1981- self ._pushdown_filters = _apply_filter_delta_bytes (self ._pushdown_filters , filter_bytes )
1982- self ._filter_delta_history .append (filter_bytes )
2056+ self ._pushdown_filters , self ._filter_delta_history , self ._filter_predicate_order = _record_filter_delta (
2057+ self ._pushdown_filters , self ._filter_delta_history , filter_bytes
2058+ )
19832059
19842060 def produce (self , out : OutputCollector , ctx : CallContext ) -> None :
19852061 """Produce the next output batch from the table function."""
@@ -2052,7 +2128,10 @@ class TableInOutExchangeState(_VgiCallStateHolder, ExchangeState):
20522128 _user_state : Annotated [Any , Transient ()] = field (default = None , repr = False )
20532129 _pushdown_filters : Annotated [Any , Transient ()] = field (default = None , repr = False ) # PushdownFilters | None
20542130 _auto_apply : Annotated [bool , Transient ()] = field (default = False , repr = False )
2131+ # The dynamic-filter deltas this stream must replay on an HTTP turn, compacted by
2132+ # _record_filter_delta, and the live predicate order they rebuild.
20552133 _filter_delta_history : list [bytes ] = field (default_factory = list , repr = False )
2134+ _filter_predicate_order : list [str ] = field (default_factory = list , repr = False )
20562135 _vgi_tracer : Annotated [VgiTracer , Transient ()] = field (default_factory = get_noop_tracer , repr = False )
20572136
20582137 def __post_init__ (self ) -> None :
@@ -2113,13 +2192,17 @@ def rehydrate(self, implementation: object) -> None:
21132192 )
21142193 self ._auto_apply = func_cls ._should_auto_apply_filters ()
21152194 if self ._call .init_call .pushdown_filters is not None :
2116- self ._pushdown_filters = func_cls .pushdown_filters (
2117- self ._call .init_call .pushdown_filters ,
2118- join_keys = self ._call .init_call .join_keys ,
2119- output_schema = self ._call .init_call .output_schema ,
2120- )
2121- for filter_bytes in self ._filter_delta_history :
2122- self ._pushdown_filters = _apply_filter_delta_bytes (self ._pushdown_filters , filter_bytes )
2195+ # Everything re-parsed here was validated when it first arrived.
2196+ with replaying_accepted_state ():
2197+ self ._pushdown_filters = _replay_filter_history (
2198+ func_cls .pushdown_filters (
2199+ self ._call .init_call .pushdown_filters ,
2200+ join_keys = self ._call .init_call .join_keys ,
2201+ output_schema = self ._call .init_call .output_schema ,
2202+ ),
2203+ self ._filter_delta_history ,
2204+ self ._filter_predicate_order ,
2205+ )
21232206 self ._params = dataclasses .replace (self ._params , current_pushdown_filters = self ._pushdown_filters )
21242207 # Restore _user_state from serialized bytes if available
21252208 if self ._user_state_bytes is not None :
@@ -2141,8 +2224,9 @@ def exchange(self, input: AnnotatedBatch, out: OutputCollector, ctx: CallContext
21412224 if len (encoded ) > (24 << 20 ):
21422225 raise ValueError ("base64 dynamic filter metadata exceeds the encoded-size limit" )
21432226 filter_bytes = base64 .b64decode (encoded , validate = True )
2144- self ._pushdown_filters = _apply_filter_delta_bytes (self ._pushdown_filters , filter_bytes )
2145- self ._filter_delta_history .append (filter_bytes )
2227+ self ._pushdown_filters , self ._filter_delta_history , self ._filter_predicate_order = _record_filter_delta (
2228+ self ._pushdown_filters , self ._filter_delta_history , filter_bytes
2229+ )
21462230 params = dataclasses .replace (
21472231 self ._params ,
21482232 auth_context = ctx .auth ,
0 commit comments