1818 _create_causal_mask_for_ring_buffer ,
1919 AttentionMHA ,
2020 KVCache ,
21+ RingKVCache ,
2122)
2223from executorch .examples .models .llama .model_args import ModelArgs
2324from executorch .examples .models .llama .rope import Rope
2425from torchao .quantization .quant_api import _replace_with_custom_fn_if_matches_filter
2526
2627
27- def _get_attention_sink_cache_size (
28- max_context_length : int ,
29- window_size : int ,
30- sink_size : int ,
31- max_seq_len : Optional [int ] = None ,
32- ) -> int :
33- """Size a sink cache for fixed sinks, one window, and one input chunk."""
34- assert sink_size >= 0 , "Attention sink size must be non-negative"
35- assert sink_size < max_context_length , (
36- f"Attention sink size ({ sink_size } ) must be smaller than the full "
37- f"context length ({ max_context_length } )"
38- )
39- assert window_size > 0 , "Sliding-window size must be positive"
40- assert sink_size + window_size <= max_context_length , (
41- f"Attention sink size ({ sink_size } ) plus sliding-window size "
42- f"({ window_size } ) cannot exceed the full context length "
43- f"({ max_context_length } )"
44- )
45- if max_seq_len is None :
46- max_seq_len = max_context_length
47- assert max_seq_len > 0 , "Maximum sequence length must be positive"
48- return sink_size + window_size + max_seq_len
49-
50-
5128class RopeWithAttentionSink (Rope ):
5229 """
5330 Rope subclass for Attention Sink models.
@@ -60,11 +37,10 @@ class RopeWithAttentionSink(Rope):
6037 - Window tokens (pos >= sink_size): wrapped into ring buffer range
6138 [sink_size, sink_size + ring_size) via modulo
6239
63- The ring buffer holds one retained window plus one maximum-size input
64- chunk. It is larger than the live window for write-ahead headroom, not to
65- keep the live window contiguous -- it can span a wrap. Across a wrap two
66- positions remap to a difference that is not their true distance, so RoPE
67- preserves relative distance only within a wrap.
40+ The ring buffer is 2x window_size for write-ahead headroom, not to keep the
41+ live window contiguous -- it can span a wrap. Across a wrap two positions
42+ remap to a difference that is not their true distance, so RoPE preserves
43+ relative distance only within a wrap.
6844 """
6945
7046 def __init__ (
@@ -76,26 +52,7 @@ def __init__(
7652 super ().__init__ (params )
7753 self .window_size = window_size
7854 self .sink_size = sink_size
79- max_seq_len = (
80- params .max_context_len
81- if getattr (params , "max_seq_len" , None ) is None
82- else int (params .max_seq_len )
83- )
84- cache_size = _get_attention_sink_cache_size (
85- params .max_context_len ,
86- window_size ,
87- sink_size ,
88- max_seq_len ,
89- )
90- self .ring_size = cache_size - sink_size
91- if self .freqs_cos .size (0 ) < cache_size :
92- freqs_cos , freqs_sin = self .precompute_freqs_cis (
93- self .params .head_dim ,
94- cache_size ,
95- self .params .rope_freq_base ,
96- )
97- self .freqs_cos = freqs_cos
98- self .freqs_sin = freqs_sin
55+ self .ring_size = window_size * 2
9956
10057 def _remap_input_pos (self , input_pos : torch .Tensor ) -> torch .Tensor :
10158 """Remap positions: sink tokens stay, window tokens wrap in ring buffer."""
@@ -176,7 +133,7 @@ class CachePositionsManagerWithSink(nn.Module):
176133 For sink_size=0: behaves exactly like original CachePositionsManager.
177134 For sink_size>0: sink tokens go to fixed positions, rest uses ring buffer.
178135
179- IMPORTANT: cache_size is the actual cache dimension, including sink slots .
136+ IMPORTANT: cache_size should be the actual cache dimension size (2x window for ring buffer) .
180137 """
181138
182139 def __init__ (self , cache_size : int , sink_size : int = 0 ):
@@ -234,7 +191,7 @@ class KVCacheWithAttentionSink(KVCache):
234191 Uses a ring buffer approach for the sliding window portion while keeping
235192 the first sink_size tokens fixed. This avoids dynamic shape operations.
236193
237- Cache layout: [fixed sink tokens ] [ring buffer for one window + one chunk ]
194+ Cache layout: [sink: 0 to sink_size-1 ] [ring_buffer: sink_size to sink_size + window_size*2 - 1 ]
238195 """
239196
240197 def __init__ (
@@ -245,27 +202,16 @@ def __init__(
245202 rope : RopeWithAttentionSink ,
246203 window_size : int ,
247204 sink_size : int ,
248- max_context_length : int ,
249- max_seq_len : Optional [int ] = None ,
250205 max_batch_size : int = 1 ,
251206 dtype = torch .float32 ,
252207 ):
253- self .full_context_length = max_context_length
254- self .max_seq_len = (
255- max_context_length if max_seq_len is None else int (max_seq_len )
256- )
257- # Keep the fixed sinks separate from the ring space needed for the
258- # retained window and the largest in-flight input chunk.
259- total_cache_size = _get_attention_sink_cache_size (
260- max_context_length ,
261- window_size ,
262- sink_size ,
263- self .max_seq_len ,
264- )
265- assert rope .ring_size == total_cache_size - sink_size , (
266- f"RoPE ring size ({ rope .ring_size } ) must match the KV-cache ring "
267- f"size ({ total_cache_size - sink_size } )"
268- )
208+ # Total cache size is sink_size + window_size * 2.
209+ # The ring buffer needs 2x the window size because at the moment a new
210+ # token is written, the previous window_size tokens must still be readable
211+ # (they haven't been overwritten yet). With only 1x, writing a new entry
212+ # would immediately evict the oldest visible token, leaving fewer than
213+ # window_size tokens available for attention.
214+ total_cache_size = sink_size + window_size * 2
269215 super ().__init__ (
270216 max_batch_size = max_batch_size ,
271217 max_context_length = total_cache_size ,
@@ -364,7 +310,6 @@ def _replace_attention(
364310 rope_with_attention_sink : RopeWithAttentionSink ,
365311 sink_size : int ,
366312 window_size : int ,
367- max_seq_len : Optional [int ],
368313):
369314 for _ , child_module in module ._modules .items ():
370315 if len (list (child_module .children ())) > 0 : # pyre-ignore [16]
@@ -373,23 +318,32 @@ def _replace_attention(
373318 rope_with_attention_sink = rope_with_attention_sink ,
374319 sink_size = sink_size ,
375320 window_size = window_size ,
376- max_seq_len = max_seq_len ,
377321 )
378322
379323 if isinstance (child_module , AttentionMHA ):
380324 kv_cache = child_module .kv_cache
381- child_module .kv_cache = KVCacheWithAttentionSink (
382- n_heads = kv_cache .n_heads ,
383- head_dim = kv_cache .head_dim ,
384- enable_dynamic_shape = kv_cache .enable_dynamic_shape ,
385- rope = rope_with_attention_sink ,
386- max_batch_size = kv_cache .max_batch_size ,
387- window_size = window_size ,
388- sink_size = sink_size ,
389- max_context_length = kv_cache .max_context_length ,
390- max_seq_len = max_seq_len ,
391- dtype = kv_cache .k_cache .dtype ,
392- )
325+ if sink_size == 0 :
326+ # No sink tokens needed — use standard RingKVCache directly
327+ child_module .kv_cache = RingKVCache (
328+ kv_cache .max_batch_size ,
329+ window_size , # RingKVCache expects user-provided window size
330+ kv_cache .n_heads ,
331+ kv_cache .head_dim ,
332+ kv_cache .enable_dynamic_shape ,
333+ kv_cache .k_cache .dtype ,
334+ )
335+ else :
336+ kv_cache_with_attention_sink = KVCacheWithAttentionSink (
337+ n_heads = kv_cache .n_heads ,
338+ head_dim = kv_cache .head_dim ,
339+ enable_dynamic_shape = kv_cache .enable_dynamic_shape ,
340+ rope = rope_with_attention_sink ,
341+ max_batch_size = kv_cache .max_batch_size ,
342+ window_size = window_size ,
343+ sink_size = sink_size ,
344+ dtype = kv_cache .k_cache .dtype ,
345+ )
346+ child_module .kv_cache = kv_cache_with_attention_sink
393347 # Don't replace forward - let the original AttentionMHA.forward handle it
394348 # since our KVCache has is_ring_buffer=True, it will use the ring buffer mask
395349
@@ -411,13 +365,11 @@ def enable_attention_sink(
411365 window_size = window_size ,
412366 sink_size = sink_size ,
413367 )
414- max_seq_len = getattr (params , "max_seq_len" , None )
415368 _replace_rope (module , rope_with_attention_sink )
416369 _replace_attention (
417370 module = module ,
418371 rope_with_attention_sink = rope_with_attention_sink ,
419372 sink_size = sink_size ,
420373 window_size = window_size ,
421- max_seq_len = max_seq_len ,
422374 )
423375 return module
0 commit comments