Skip to content

Commit 14dab10

Browse files
authored
Apply local sliding-window mask in StaticAttentionIOManager single-chunk prefill (pytorch#21384)
Differential Revision: D113574455 Pull Request resolved: pytorch#21384
1 parent d1df092 commit 14dab10

1 file changed

Lines changed: 25 additions & 4 deletions

File tree

‎examples/models/llama/static_attention.py‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -429,13 +429,34 @@ def prefill(
429429
if self.cache_full:
430430
raise RuntimeError("KV cache is full.")
431431

432+
# Global (full-attention) layers use the largest cache; local
433+
# (sliding-window) layers use a strictly smaller cache. In a global-only
434+
# model every layer shares the same cache, so this stays equal to it.
435+
global_cache_len = max(
436+
(mask.cache_len for mask in self._masks.values()), default=0
437+
)
432438
for mask in self._masks.values():
433-
mask.set_input_mask(
434-
torch.triu(
439+
input_mask = torch.triu(
440+
torch.full((1, self.input_len, self.input_len), self.mask_val),
441+
diagonal=1,
442+
)
443+
# Local (sliding-window) layers use a per-layer cache_len smaller than
444+
# the prompt. When the whole prompt is prefilled in a single chunk, the
445+
# cache-eviction window is never exercised, so also mask keys older than
446+
# the window here to reproduce the local attention instead of full causal.
447+
# Only genuine local layers (cache smaller than the global cache) are
448+
# windowed; a global layer is left full-causal even when its (nominal)
449+
# cache is smaller than the prompt. No-op for chunked prefill
450+
# (input_len <= cache_len).
451+
if (
452+
0 < mask.cache_len < self.input_len
453+
and mask.cache_len < global_cache_len
454+
):
455+
input_mask = input_mask + torch.tril(
435456
torch.full((1, self.input_len, self.input_len), self.mask_val),
436-
diagonal=1,
457+
diagonal=-mask.cache_len,
437458
)
438-
)
459+
mask.set_input_mask(input_mask)
439460

440461
if isinstance(tokens, list):
441462
tokens = torch.tensor([tokens], dtype=torch.int32)

0 commit comments

Comments
 (0)