@@ -205,10 +205,9 @@ def unmask(self, new_unmasked_len):
205205 self .tensor [
206206 :,
207207 :,
208- max (
209- 0 , self .cache_len - self .unmasked_len - new_unmasked_len
210- ) : self .cache_len
211- - self .unmasked_len ,
208+ max (0 , self .cache_len - self .unmasked_len - new_unmasked_len ) : max (
209+ 0 , self .cache_len - self .unmasked_len
210+ ),
212211 ] = 0
213212
214213 if self .style == "smart_mask" :
@@ -274,6 +273,12 @@ def __init__(
274273 )
275274 for cl in set (cache_lens )
276275 }
276+ # Global (full-attention) layers use the largest cache; local
277+ # (sliding-window) layers use a strictly smaller cache. In a global-only
278+ # model every layer shares the same cache, so this stays equal to it.
279+ self ._global_cache_len = max (
280+ (mask .cache_len for mask in self ._masks .values ()), default = 0
281+ )
277282
278283 if isinstance (config_or_model , ModelArgs ):
279284 self ._from_config (config_or_model , cache_lens_dict , batch_size , dtype )
@@ -421,6 +426,40 @@ def reset(self):
421426 for mask in self ._masks .values ():
422427 mask .reset ()
423428
429+ def _is_windowed_mask (self , mask : StaticAttentionMask ) -> bool :
430+ """
431+ A mask belongs to a local (sliding-window) layer iff its cache is strictly
432+ smaller than the global cache. A pure sliding-window model is
433+ indistinguishable from a global one by cache_len alone and counts as global.
434+ """
435+ return 0 < mask .cache_len < self ._global_cache_len
436+
437+ def _mask_cache_outside_window (self ):
438+ """
439+ Rewrite the cache region of every windowed mask so each query row only sees
440+ its own window. `unmask` fills the cache region identically for all rows,
441+ which lets query row k attend to cache_len + k + 1 keys instead of cache_len.
442+
443+ With shift_pointer the valid cache entries are right aligned and ordered
444+ oldest to newest, so with W = cache_len and v = min(pos, W) valid entries,
445+ cache column c is visible to query row k iff c >= max(W - v, k + 1). This
446+ holds for both prefill chunks and decode steps, and composes with the
447+ in-chunk causal/window mask set by the caller.
448+ """
449+ for mask in self ._masks .values ():
450+ if not self ._is_windowed_mask (mask ):
451+ continue
452+ w = mask .cache_len
453+ first_valid = w - min (self .pos , w )
454+ rows = torch .arange (self .input_len ).unsqueeze (1 )
455+ cols = torch .arange (w ).unsqueeze (0 )
456+ visible = cols >= torch .clamp (rows + 1 , min = first_valid )
457+ cache_mask = torch .full (
458+ (self .input_len , w ), mask .mask_val , dtype = mask .tensor .dtype
459+ )
460+ cache_mask [visible ] = 0
461+ mask .tensor [:, :, :w ] = cache_mask
462+
424463 def prefill (
425464 self ,
426465 model : Callable [..., Any ],
@@ -429,12 +468,6 @@ def prefill(
429468 if self .cache_full :
430469 raise RuntimeError ("KV cache is full." )
431470
432- # Global (full-attention) layers use the largest cache; local
433- # (sliding-window) layers use a strictly smaller cache. In a global-only
434- # model every layer shares the same cache, so this stays equal to it.
435- global_cache_len = max (
436- (mask .cache_len for mask in self ._masks .values ()), default = 0
437- )
438471 for mask in self ._masks .values ():
439472 input_mask = torch .triu (
440473 torch .full ((1 , self .input_len , self .input_len ), self .mask_val ),
@@ -448,10 +481,7 @@ def prefill(
448481 # windowed; a global layer is left full-causal even when its (nominal)
449482 # cache is smaller than the prompt. No-op for chunked prefill
450483 # (input_len <= cache_len).
451- if (
452- 0 < mask .cache_len < self .input_len
453- and mask .cache_len < global_cache_len
454- ):
484+ if self ._is_windowed_mask (mask ) and mask .cache_len < self .input_len :
455485 input_mask = input_mask + torch .tril (
456486 torch .full ((1 , self .input_len , self .input_len ), self .mask_val ),
457487 diagonal = - mask .cache_len ,
@@ -464,6 +494,7 @@ def prefill(
464494 logits = None
465495 all_logits = None
466496 for i in range (0 , tokens .size (1 ), self .input_len ):
497+ self ._mask_cache_outside_window ()
467498 logits = self ._run_once (model , tokens [:, i : i + self .input_len ])[0 ]
468499 if self .generate_full_logits :
469500 if all_logits is None :
@@ -497,6 +528,7 @@ def decode(
497528 stop_tokens = stop_tokens or []
498529 new_tokens = [init_token ]
499530 for _ in range (n ):
531+ self ._mask_cache_outside_window ()
500532 y = self ._run_once (model , new_tokens [- 1 :])[0 ]
501533 if self .generate_full_logits :
502534 new_tokens .append (y [:, :1 , ...].argmax ().item ())
0 commit comments