@@ -429,13 +429,34 @@ def prefill(
429429 if self .cache_full :
430430 raise RuntimeError ("KV cache is full." )
431431
432+ # Global (full-attention) layers use the largest cache; local
433+ # (sliding-window) layers use a strictly smaller cache. In a global-only
434+ # model every layer shares the same cache, so this stays equal to it.
435+ global_cache_len = max (
436+ (mask .cache_len for mask in self ._masks .values ()), default = 0
437+ )
432438 for mask in self ._masks .values ():
433- mask .set_input_mask (
434- torch .triu (
439+ input_mask = torch .triu (
440+ torch .full ((1 , self .input_len , self .input_len ), self .mask_val ),
441+ diagonal = 1 ,
442+ )
443+ # Local (sliding-window) layers use a per-layer cache_len smaller than
444+ # the prompt. When the whole prompt is prefilled in a single chunk, the
445+ # cache-eviction window is never exercised, so also mask keys older than
446+ # the window here to reproduce the local attention instead of full causal.
447+ # Only genuine local layers (cache smaller than the global cache) are
448+ # windowed; a global layer is left full-causal even when its (nominal)
449+ # cache is smaller than the prompt. No-op for chunked prefill
450+ # (input_len <= cache_len).
451+ if (
452+ 0 < mask .cache_len < self .input_len
453+ and mask .cache_len < global_cache_len
454+ ):
455+ input_mask = input_mask + torch .tril (
435456 torch .full ((1 , self .input_len , self .input_len ), self .mask_val ),
436- diagonal = 1 ,
457+ diagonal = - mask . cache_len ,
437458 )
438- )
459+ mask . set_input_mask ( input_mask )
439460
440461 if isinstance (tokens , list ):
441462 tokens = torch .tensor ([tokens ], dtype = torch .int32 )
0 commit comments