Skip to content

Commit 3caaaca

Browse files
authored
Fix sliding-window over-attention in chunked prefill (pytorch#21857)
Differential Revision: D116080453 Pull Request resolved: pytorch#21857
1 parent 5978509 commit 3caaaca

2 files changed

Lines changed: 93 additions & 14 deletions

File tree

‎examples/models/llama/static_attention.py‎

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -205,10 +205,9 @@ def unmask(self, new_unmasked_len):
205205
self.tensor[
206206
:,
207207
:,
208-
max(
209-
0, self.cache_len - self.unmasked_len - new_unmasked_len
210-
) : self.cache_len
211-
- self.unmasked_len,
208+
max(0, self.cache_len - self.unmasked_len - new_unmasked_len) : max(
209+
0, self.cache_len - self.unmasked_len
210+
),
212211
] = 0
213212

214213
if self.style == "smart_mask":
@@ -274,6 +273,12 @@ def __init__(
274273
)
275274
for cl in set(cache_lens)
276275
}
276+
# Global (full-attention) layers use the largest cache; local
277+
# (sliding-window) layers use a strictly smaller cache. In a global-only
278+
# model every layer shares the same cache, so this stays equal to it.
279+
self._global_cache_len = max(
280+
(mask.cache_len for mask in self._masks.values()), default=0
281+
)
277282

278283
if isinstance(config_or_model, ModelArgs):
279284
self._from_config(config_or_model, cache_lens_dict, batch_size, dtype)
@@ -421,6 +426,40 @@ def reset(self):
421426
for mask in self._masks.values():
422427
mask.reset()
423428

429+
def _is_windowed_mask(self, mask: StaticAttentionMask) -> bool:
430+
"""
431+
A mask belongs to a local (sliding-window) layer iff its cache is strictly
432+
smaller than the global cache. A pure sliding-window model is
433+
indistinguishable from a global one by cache_len alone and counts as global.
434+
"""
435+
return 0 < mask.cache_len < self._global_cache_len
436+
437+
def _mask_cache_outside_window(self):
438+
"""
439+
Rewrite the cache region of every windowed mask so each query row only sees
440+
its own window. `unmask` fills the cache region identically for all rows,
441+
which lets query row k attend to cache_len + k + 1 keys instead of cache_len.
442+
443+
With shift_pointer the valid cache entries are right aligned and ordered
444+
oldest to newest, so with W = cache_len and v = min(pos, W) valid entries,
445+
cache column c is visible to query row k iff c >= max(W - v, k + 1). This
446+
holds for both prefill chunks and decode steps, and composes with the
447+
in-chunk causal/window mask set by the caller.
448+
"""
449+
for mask in self._masks.values():
450+
if not self._is_windowed_mask(mask):
451+
continue
452+
w = mask.cache_len
453+
first_valid = w - min(self.pos, w)
454+
rows = torch.arange(self.input_len).unsqueeze(1)
455+
cols = torch.arange(w).unsqueeze(0)
456+
visible = cols >= torch.clamp(rows + 1, min=first_valid)
457+
cache_mask = torch.full(
458+
(self.input_len, w), mask.mask_val, dtype=mask.tensor.dtype
459+
)
460+
cache_mask[visible] = 0
461+
mask.tensor[:, :, :w] = cache_mask
462+
424463
def prefill(
425464
self,
426465
model: Callable[..., Any],
@@ -429,12 +468,6 @@ def prefill(
429468
if self.cache_full:
430469
raise RuntimeError("KV cache is full.")
431470

432-
# Global (full-attention) layers use the largest cache; local
433-
# (sliding-window) layers use a strictly smaller cache. In a global-only
434-
# model every layer shares the same cache, so this stays equal to it.
435-
global_cache_len = max(
436-
(mask.cache_len for mask in self._masks.values()), default=0
437-
)
438471
for mask in self._masks.values():
439472
input_mask = torch.triu(
440473
torch.full((1, self.input_len, self.input_len), self.mask_val),
@@ -448,10 +481,7 @@ def prefill(
448481
# windowed; a global layer is left full-causal even when its (nominal)
449482
# cache is smaller than the prompt. No-op for chunked prefill
450483
# (input_len <= cache_len).
451-
if (
452-
0 < mask.cache_len < self.input_len
453-
and mask.cache_len < global_cache_len
454-
):
484+
if self._is_windowed_mask(mask) and mask.cache_len < self.input_len:
455485
input_mask = input_mask + torch.tril(
456486
torch.full((1, self.input_len, self.input_len), self.mask_val),
457487
diagonal=-mask.cache_len,
@@ -464,6 +494,7 @@ def prefill(
464494
logits = None
465495
all_logits = None
466496
for i in range(0, tokens.size(1), self.input_len):
497+
self._mask_cache_outside_window()
467498
logits = self._run_once(model, tokens[:, i : i + self.input_len])[0]
468499
if self.generate_full_logits:
469500
if all_logits is None:
@@ -497,6 +528,7 @@ def decode(
497528
stop_tokens = stop_tokens or []
498529
new_tokens = [init_token]
499530
for _ in range(n):
531+
self._mask_cache_outside_window()
500532
y = self._run_once(model, new_tokens[-1:])[0]
501533
if self.generate_full_logits:
502534
new_tokens.append(y[:, :1, ...].argmax().item())

‎examples/models/llama/tests/test_static_attention.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,53 @@ def test(style, attention_type):
258258
):
259259
test(*args)
260260

261+
def test_sliding_window_chunked_prefill(self):
262+
prompt_len = 24
263+
global_cache_len = 32
264+
265+
def sliding_window_mask(window):
266+
causal = torch.tril(torch.ones(prompt_len, prompt_len, dtype=torch.bool))
267+
return causal & torch.triu(
268+
torch.ones(prompt_len, prompt_len, dtype=torch.bool),
269+
diagonal=-(window - 1),
270+
)
271+
272+
def test(window, chunk_len):
273+
config = ModelArgs(
274+
dim=64,
275+
n_heads=4,
276+
n_kv_heads=2,
277+
max_seq_len=prompt_len,
278+
max_context_len=global_cache_len + prompt_len,
279+
n_layers=2,
280+
vocab_size=128,
281+
generate_full_logits=True,
282+
)
283+
mha_transformer, static_transformer, static_config = (
284+
self._get_test_transformers(config)
285+
)
286+
# Layer 0 is the local (sliding-window) layer, layer 1 is global.
287+
mha_transformer.layers[0].attention.mask[:prompt_len, :prompt_len] = (
288+
sliding_window_mask(window)
289+
)
290+
x = torch.randint(config.vocab_size, (1, prompt_len))
291+
expected = mha_transformer(x)
292+
293+
cache_lens = [window, global_cache_len]
294+
single_chunk = StaticAttentionIOManager(
295+
static_config, prompt_len, cache_lens
296+
).prefill(static_transformer, x)
297+
chunked = StaticAttentionIOManager(
298+
static_config, chunk_len, cache_lens
299+
).prefill(static_transformer, x)
300+
301+
msg = f"window={window}, chunk_len={chunk_len}"
302+
self.assertTrue(torch.isclose(chunked, single_chunk, rtol=1e-3).all(), msg)
303+
self.assertTrue(torch.isclose(chunked, expected, rtol=1e-3).all(), msg)
304+
305+
test(window=4, chunk_len=8) # chunk longer than the window
306+
test(window=12, chunk_len=6) # chunk shorter than the window
307+
261308
def test_lookahead_decode(self):
262309
config = ModelArgs(
263310
dim=64,

0 commit comments

Comments
 (0)