2929
3030from dataclasses import dataclass
3131from enum import Enum
32- from typing import List , Mapping , Optional , Sequence , Set , Tuple
32+ from typing import Dict , List , Mapping , Optional , Sequence , Set , Tuple
3333
3434import torch
3535import torch .nn .functional as F
@@ -54,6 +54,33 @@ class AttendSpec:
5454 mask : Optional [torch .Tensor ] = None # EXPLICIT only: bool, true = attend
5555
5656
57+ class LayerKind (Enum ):
58+ FLAT = "flat" # retains all history
59+ RING = "ring" # sliding window over the newest `window` positions
60+
61+
62+ @dataclass (frozen = True )
63+ class LayerPolicy :
64+ """Per-layer cache kind and its parameters."""
65+
66+ kind : LayerKind = LayerKind .FLAT
67+ window : int = 0 # RING only: window size in positions
68+
69+ def __post_init__ (self ):
70+ if self .kind is LayerKind .RING and self .window <= 0 :
71+ raise ValueError ("a ring layer needs a positive window" )
72+ if self .kind is LayerKind .FLAT and self .window != 0 :
73+ raise ValueError ("a flat layer retains all history; window must be 0" )
74+
75+ @classmethod
76+ def flat (cls ) -> "LayerPolicy" :
77+ return cls (kind = LayerKind .FLAT )
78+
79+ @classmethod
80+ def ring (cls , window : int ) -> "LayerPolicy" :
81+ return cls (kind = LayerKind .RING , window = window )
82+
83+
5784@experimental (
5885 "update_and_attend KV cache is experimental and may change without notice."
5986)
@@ -66,10 +93,17 @@ class CacheConfig:
6693 sizing : CacheSizing = CacheSizing .DYNAMIC
6794 dtype : torch .dtype = torch .float32
6895 batch_size : int = 1
96+ # Per-layer policy: one entry applies to every layer, else one per layer.
97+ layers : Sequence [LayerPolicy ] = (LayerPolicy .flat (),)
6998
7099 def __post_init__ (self ):
71100 if self .capacity <= 0 :
72101 raise ValueError ("capacity must be positive" )
102+ if len (self .layers ) not in (1 , self .n_layers ):
103+ raise ValueError ("layers must be one policy, or one per layer" )
104+
105+ def policy_for (self , layer_id : int ) -> LayerPolicy :
106+ return self .layers [0 ] if len (self .layers ) == 1 else self .layers [layer_id ]
73107
74108
75109@experimental (
@@ -154,23 +188,33 @@ def update_and_fetch(
154188 v_hist = self ._v [layer_id ]
155189 self ._used [layer_id ] = new_used
156190
157- return k_hist , v_hist , self ._spec (q_len , new_used , k .device )
191+ return k_hist , v_hist , self ._spec (layer_id , q_len , new_used , k .device )
158192
159- @staticmethod
160- def _spec (q_len : int , total : int , device : torch .device ) -> AttendSpec :
161- """The mask semantic for q_len new cells at the tail of a total window."""
162- if q_len == 1 :
193+ def _spec (
194+ self , layer_id : int , q_len : int , total : int , device : torch .device
195+ ) -> AttendSpec :
196+ """The mask semantic for q_len new cells at the tail of a total window.
197+
198+ The new cells are at the tail, so query i attends keys up to
199+ ``i + total - q_len``, and a sliding window bounds it from below at
200+ ``i + total - q_len - window``. Whichever bound the fused kinds cannot
201+ express is what makes the step EXPLICIT.
202+ """
203+ window = self .config .policy_for (layer_id ).window
204+ windowed = 0 < window < total
205+ if q_len == 1 and not windowed :
163206 return AttendSpec (kind = MaskKind .NONE )
164- if q_len == total :
207+ if q_len == total and not windowed :
165208 return AttendSpec (kind = MaskKind .CAUSAL )
166- # Continuation (chunked / multi-turn): the new cells are at the tail, so
167- # query i attends keys up to i + total - q_len. A backend whose causal is
168- # lower-right aligned fuses that as CAUSAL; torch's is_causal
169- # is upper-left, so the reference hands back the band itself.
209+ # torch's is_causal is upper-left and expresses no window, so the band
210+ # is handed back explicitly.
170211 offsets = torch .arange (total , device = device ) - torch .arange (
171212 q_len , device = device
172213 ).unsqueeze (- 1 )
173- return AttendSpec (kind = MaskKind .EXPLICIT , mask = offsets <= total - q_len )
214+ band = offsets <= total - q_len
215+ if windowed :
216+ band &= offsets > total - q_len - window
217+ return AttendSpec (kind = MaskKind .EXPLICIT , mask = band )
174218
175219
176220# A cell's owners are a bitset in a torch int64, so bit 63 (the sign bit) is out.
@@ -217,10 +261,24 @@ def flatten_step(
217261
218262@dataclass
219263class _CellStepPlan :
220- """One step's allocation, shared by every layer of that forward."""
264+ """One step's allocation, shared by every layer of that forward.
265+
266+ Layers can differ in window, so the mask is per *policy* rather than per
267+ layer: membership and causality are common, and only the lower bound moves.
268+ `masks` memoizes one per distinct window (0 = unwindowed) as layers ask, so
269+ a mixed model costs two masks a step rather than one per layer.
270+ """
221271
222272 cells : torch .Tensor # [n_tok] long -- the cell each query token was given
223- mask : torch .Tensor # [n_tok, read_len] bool -- true = attend
273+ base : torch .Tensor # [n_tok, read_len] bool -- occupied, same seq, causal
274+ cell_pos : torch .Tensor # [read_len] -- the window's positions
275+ tok_pos : torch .Tensor # [n_tok, 1] -- this step's positions
276+ masks : Dict [int , torch .Tensor ] # window -> mask; 0 is `base` itself
277+
278+ def mask_for (self , window : int ) -> torch .Tensor :
279+ if window not in self .masks :
280+ self .masks [window ] = self .base & (self .cell_pos > self .tok_pos - window )
281+ return self .masks [window ]
224282
225283
226284@experimental (
@@ -233,15 +291,20 @@ class CellReferenceCache:
233291 sequences owning it, so a sequence need not be contiguous and two may share
234292 cells -- a fork sets a second bit instead of copying K/V. Visibility is then
235293 a property of the cell rather than of the layout: query i attends cell j iff
236- j is occupied, shares a sequence with i, and is no newer than i. No causal
237- alignment can express that, so the spec is always EXPLICIT.
294+ j is occupied, shares a sequence with i, and is no newer than i -- and, on a
295+ windowed layer, no older than its window. No causal alignment can express
296+ that, so the spec is always EXPLICIT.
238297
239298 The batch is flat: tokens from every sequence sit on one axis with B = 1,
240299 and sequence identity is supplied out-of-band. ``begin_step`` declares which
241300 sequence each of the next forward's tokens belongs to; the positions arrive
242301 with the forward itself, in the op's ``position`` tensor, so cells are
243302 allocated on the first layer of the step and memoized for the rest of it.
244303
304+ Layers may window differently: the mask is memoized per window rather than
305+ per layer, over cells they all share. Nothing is evicted -- reclaiming under
306+ mixed windows needs one cache per policy group.
307+
245308 DYNAMIC sizing grows the pool to the occupied extent, so a short session
246309 reserves a short pool rather than the whole context. Growth must keep every
247310 cell's index and its bytes -- a cell's index is its name, held by the plan
@@ -383,15 +446,18 @@ def update_and_fetch(
383446 self ._plan = self ._allocate (position )
384447 self ._served .add (layer_id )
385448
386- read_len = self ._plan .mask .shape [- 1 ]
449+ read_len = self ._plan .base .shape [- 1 ]
387450 self ._ensure (layer_id , read_len )
388451 cells = self ._plan .cells
389452 self ._k [layer_id ][:, :, cells , :] = k .to (self .config .dtype )
390453 self ._v [layer_id ][:, :, cells , :] = v .to (self .config .dtype )
391454 return (
392455 self ._k [layer_id ][:, :, :read_len , :],
393456 self ._v [layer_id ][:, :, :read_len , :],
394- AttendSpec (kind = MaskKind .EXPLICIT , mask = self ._plan .mask ),
457+ AttendSpec (
458+ kind = MaskKind .EXPLICIT ,
459+ mask = self ._plan .mask_for (self .config .policy_for (layer_id ).window ),
460+ ),
395461 )
396462
397463 # -- internals ----------------------------------------------------------
@@ -419,19 +485,24 @@ def _allocate(self, position: torch.Tensor) -> _CellStepPlan:
419485 self ._claim (pos , 1 << seq_id )
420486 for pos , seq_id in zip (positions , self ._step_seq_ids )
421487 ]
422- # Occupied, sharing a sequence, and no newer than the query. The step's
423- # own cells are already placed, so a query sees itself and any earlier
424- # token of its sequence in the same batch.
488+ # Occupied, sharing a sequence, and no newer than the query -- plus, when
489+ # windowed, no older than its window. The step's own cells are already
490+ # placed, so a query sees itself and any earlier token of its sequence in
491+ # the same batch.
425492 n = self ._used_end
426493 cell_pos = torch .tensor (self ._pos [:n ], device = device )
427494 cell_owners = torch .tensor (self ._owners [:n ], device = device )
428495 tok_pos = torch .tensor (positions , device = device ).unsqueeze (- 1 )
429496 tok_bit = torch .tensor (
430497 [1 << seq_id for seq_id in self ._step_seq_ids ], device = device
431498 ).unsqueeze (- 1 )
432- mask = (cell_pos >= 0 ) & ((cell_owners & tok_bit ) != 0 ) & (cell_pos <= tok_pos )
499+ base = (cell_pos >= 0 ) & ((cell_owners & tok_bit ) != 0 ) & (cell_pos <= tok_pos )
433500 return _CellStepPlan (
434- cells = torch .tensor (cells , dtype = torch .long , device = device ), mask = mask
501+ cells = torch .tensor (cells , dtype = torch .long , device = device ),
502+ base = base ,
503+ cell_pos = cell_pos ,
504+ tok_pos = tok_pos ,
505+ masks = {0 : base },
435506 )
436507
437508 def _ensure (self , layer_id : int , rows : int ) -> None :
0 commit comments