Skip to content

Commit 80c7087

Browse files
Add sliding windows to the off-graph KV-cache reference (pytorch#21901)
## Summary Adds a sliding window to the eager KV-cache reference. `CacheConfig.window` takes an int for every layer or one entry per layer, so a model that alternates policies (gemma-style) is expressible. Both reference caches honour it. `ContiguousReferenceCache` adds a lower bound to the band it already emits, and windowing alone is now enough to make a step EXPLICIT. `CellReferenceCache` memoizes the mask per *window* rather than per layer: occupancy, membership and causality are common to every layer of a step, and only the lower bound moves, so a mixed model pays two masks per step instead of one per layer. The reference masks; it does not evict. Cells stay allocated under a window, because reclaiming them when layers disagree needs one cache per policy group. ## Files - `extension/llm/cache/reference_cache.py` — `CacheConfig.window` + `window_for`, the windowed band in `ContiguousReferenceCache._spec`, and `_CellStepPlan` split into a shared base plus a per-window memo. - `extension/llm/cache/test_update_and_attend.py` — 8 new tests. ## Testing `pytest extension/llm/cache/test_update_and_attend.py` New coverage: a window narrowing each query without crossing sequences, layers windowing independently, layers sharing a window sharing one mask, windowed decode attending only retained cells, and a window wider than the history staying fused.
1 parent 910f816 commit 80c7087

2 files changed

Lines changed: 334 additions & 30 deletions

File tree

‎extension/llm/cache/reference_cache.py‎

Lines changed: 94 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929

3030
from dataclasses import dataclass
3131
from enum import Enum
32-
from typing import List, Mapping, Optional, Sequence, Set, Tuple
32+
from typing import Dict, List, Mapping, Optional, Sequence, Set, Tuple
3333

3434
import torch
3535
import torch.nn.functional as F
@@ -54,6 +54,33 @@ class AttendSpec:
5454
mask: Optional[torch.Tensor] = None # EXPLICIT only: bool, true = attend
5555

5656

57+
class LayerKind(Enum):
58+
FLAT = "flat" # retains all history
59+
RING = "ring" # sliding window over the newest `window` positions
60+
61+
62+
@dataclass(frozen=True)
63+
class LayerPolicy:
64+
"""Per-layer cache kind and its parameters."""
65+
66+
kind: LayerKind = LayerKind.FLAT
67+
window: int = 0 # RING only: window size in positions
68+
69+
def __post_init__(self):
70+
if self.kind is LayerKind.RING and self.window <= 0:
71+
raise ValueError("a ring layer needs a positive window")
72+
if self.kind is LayerKind.FLAT and self.window != 0:
73+
raise ValueError("a flat layer retains all history; window must be 0")
74+
75+
@classmethod
76+
def flat(cls) -> "LayerPolicy":
77+
return cls(kind=LayerKind.FLAT)
78+
79+
@classmethod
80+
def ring(cls, window: int) -> "LayerPolicy":
81+
return cls(kind=LayerKind.RING, window=window)
82+
83+
5784
@experimental(
5885
"update_and_attend KV cache is experimental and may change without notice."
5986
)
@@ -66,10 +93,17 @@ class CacheConfig:
6693
sizing: CacheSizing = CacheSizing.DYNAMIC
6794
dtype: torch.dtype = torch.float32
6895
batch_size: int = 1
96+
# Per-layer policy: one entry applies to every layer, else one per layer.
97+
layers: Sequence[LayerPolicy] = (LayerPolicy.flat(),)
6998

7099
def __post_init__(self):
71100
if self.capacity <= 0:
72101
raise ValueError("capacity must be positive")
102+
if len(self.layers) not in (1, self.n_layers):
103+
raise ValueError("layers must be one policy, or one per layer")
104+
105+
def policy_for(self, layer_id: int) -> LayerPolicy:
106+
return self.layers[0] if len(self.layers) == 1 else self.layers[layer_id]
73107

74108

75109
@experimental(
@@ -154,23 +188,33 @@ def update_and_fetch(
154188
v_hist = self._v[layer_id]
155189
self._used[layer_id] = new_used
156190

157-
return k_hist, v_hist, self._spec(q_len, new_used, k.device)
191+
return k_hist, v_hist, self._spec(layer_id, q_len, new_used, k.device)
158192

159-
@staticmethod
160-
def _spec(q_len: int, total: int, device: torch.device) -> AttendSpec:
161-
"""The mask semantic for q_len new cells at the tail of a total window."""
162-
if q_len == 1:
193+
def _spec(
194+
self, layer_id: int, q_len: int, total: int, device: torch.device
195+
) -> AttendSpec:
196+
"""The mask semantic for q_len new cells at the tail of a total window.
197+
198+
The new cells are at the tail, so query i attends keys up to
199+
``i + total - q_len``, and a sliding window bounds it from below at
200+
``i + total - q_len - window``. Whichever bound the fused kinds cannot
201+
express is what makes the step EXPLICIT.
202+
"""
203+
window = self.config.policy_for(layer_id).window
204+
windowed = 0 < window < total
205+
if q_len == 1 and not windowed:
163206
return AttendSpec(kind=MaskKind.NONE)
164-
if q_len == total:
207+
if q_len == total and not windowed:
165208
return AttendSpec(kind=MaskKind.CAUSAL)
166-
# Continuation (chunked / multi-turn): the new cells are at the tail, so
167-
# query i attends keys up to i + total - q_len. A backend whose causal is
168-
# lower-right aligned fuses that as CAUSAL; torch's is_causal
169-
# is upper-left, so the reference hands back the band itself.
209+
# torch's is_causal is upper-left and expresses no window, so the band
210+
# is handed back explicitly.
170211
offsets = torch.arange(total, device=device) - torch.arange(
171212
q_len, device=device
172213
).unsqueeze(-1)
173-
return AttendSpec(kind=MaskKind.EXPLICIT, mask=offsets <= total - q_len)
214+
band = offsets <= total - q_len
215+
if windowed:
216+
band &= offsets > total - q_len - window
217+
return AttendSpec(kind=MaskKind.EXPLICIT, mask=band)
174218

175219

176220
# A cell's owners are a bitset in a torch int64, so bit 63 (the sign bit) is out.
@@ -217,10 +261,24 @@ def flatten_step(
217261

218262
@dataclass
219263
class _CellStepPlan:
220-
"""One step's allocation, shared by every layer of that forward."""
264+
"""One step's allocation, shared by every layer of that forward.
265+
266+
Layers can differ in window, so the mask is per *policy* rather than per
267+
layer: membership and causality are common, and only the lower bound moves.
268+
`masks` memoizes one per distinct window (0 = unwindowed) as layers ask, so
269+
a mixed model costs two masks a step rather than one per layer.
270+
"""
221271

222272
cells: torch.Tensor # [n_tok] long -- the cell each query token was given
223-
mask: torch.Tensor # [n_tok, read_len] bool -- true = attend
273+
base: torch.Tensor # [n_tok, read_len] bool -- occupied, same seq, causal
274+
cell_pos: torch.Tensor # [read_len] -- the window's positions
275+
tok_pos: torch.Tensor # [n_tok, 1] -- this step's positions
276+
masks: Dict[int, torch.Tensor] # window -> mask; 0 is `base` itself
277+
278+
def mask_for(self, window: int) -> torch.Tensor:
279+
if window not in self.masks:
280+
self.masks[window] = self.base & (self.cell_pos > self.tok_pos - window)
281+
return self.masks[window]
224282

225283

226284
@experimental(
@@ -233,15 +291,20 @@ class CellReferenceCache:
233291
sequences owning it, so a sequence need not be contiguous and two may share
234292
cells -- a fork sets a second bit instead of copying K/V. Visibility is then
235293
a property of the cell rather than of the layout: query i attends cell j iff
236-
j is occupied, shares a sequence with i, and is no newer than i. No causal
237-
alignment can express that, so the spec is always EXPLICIT.
294+
j is occupied, shares a sequence with i, and is no newer than i -- and, on a
295+
windowed layer, no older than its window. No causal alignment can express
296+
that, so the spec is always EXPLICIT.
238297
239298
The batch is flat: tokens from every sequence sit on one axis with B = 1,
240299
and sequence identity is supplied out-of-band. ``begin_step`` declares which
241300
sequence each of the next forward's tokens belongs to; the positions arrive
242301
with the forward itself, in the op's ``position`` tensor, so cells are
243302
allocated on the first layer of the step and memoized for the rest of it.
244303
304+
Layers may window differently: the mask is memoized per window rather than
305+
per layer, over cells they all share. Nothing is evicted -- reclaiming under
306+
mixed windows needs one cache per policy group.
307+
245308
DYNAMIC sizing grows the pool to the occupied extent, so a short session
246309
reserves a short pool rather than the whole context. Growth must keep every
247310
cell's index and its bytes -- a cell's index is its name, held by the plan
@@ -383,15 +446,18 @@ def update_and_fetch(
383446
self._plan = self._allocate(position)
384447
self._served.add(layer_id)
385448

386-
read_len = self._plan.mask.shape[-1]
449+
read_len = self._plan.base.shape[-1]
387450
self._ensure(layer_id, read_len)
388451
cells = self._plan.cells
389452
self._k[layer_id][:, :, cells, :] = k.to(self.config.dtype)
390453
self._v[layer_id][:, :, cells, :] = v.to(self.config.dtype)
391454
return (
392455
self._k[layer_id][:, :, :read_len, :],
393456
self._v[layer_id][:, :, :read_len, :],
394-
AttendSpec(kind=MaskKind.EXPLICIT, mask=self._plan.mask),
457+
AttendSpec(
458+
kind=MaskKind.EXPLICIT,
459+
mask=self._plan.mask_for(self.config.policy_for(layer_id).window),
460+
),
395461
)
396462

397463
# -- internals ----------------------------------------------------------
@@ -419,19 +485,24 @@ def _allocate(self, position: torch.Tensor) -> _CellStepPlan:
419485
self._claim(pos, 1 << seq_id)
420486
for pos, seq_id in zip(positions, self._step_seq_ids)
421487
]
422-
# Occupied, sharing a sequence, and no newer than the query. The step's
423-
# own cells are already placed, so a query sees itself and any earlier
424-
# token of its sequence in the same batch.
488+
# Occupied, sharing a sequence, and no newer than the query -- plus, when
489+
# windowed, no older than its window. The step's own cells are already
490+
# placed, so a query sees itself and any earlier token of its sequence in
491+
# the same batch.
425492
n = self._used_end
426493
cell_pos = torch.tensor(self._pos[:n], device=device)
427494
cell_owners = torch.tensor(self._owners[:n], device=device)
428495
tok_pos = torch.tensor(positions, device=device).unsqueeze(-1)
429496
tok_bit = torch.tensor(
430497
[1 << seq_id for seq_id in self._step_seq_ids], device=device
431498
).unsqueeze(-1)
432-
mask = (cell_pos >= 0) & ((cell_owners & tok_bit) != 0) & (cell_pos <= tok_pos)
499+
base = (cell_pos >= 0) & ((cell_owners & tok_bit) != 0) & (cell_pos <= tok_pos)
433500
return _CellStepPlan(
434-
cells=torch.tensor(cells, dtype=torch.long, device=device), mask=mask
501+
cells=torch.tensor(cells, dtype=torch.long, device=device),
502+
base=base,
503+
cell_pos=cell_pos,
504+
tok_pos=tok_pos,
505+
masks={0: base},
435506
)
436507

437508
def _ensure(self, layer_id: int, rows: int) -> None:

0 commit comments

Comments
 (0)