Skip to content

Commit a33ac10

Browse files
authored
1 parent 25f155c commit a33ac10

10 files changed

Lines changed: 181 additions & 794 deletions

File tree

‎examples/models/llama/attention.py‎

Lines changed: 6 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -262,23 +262,6 @@ def calculate_positions_and_update_indices(self, input_pos: torch.Tensor, seq_le
262262
return indices
263263

264264

265-
def _get_ring_cache_size(
266-
max_context_length: int,
267-
window_size: int,
268-
max_seq_len: Optional[int] = None,
269-
) -> int:
270-
"""Size an SWA cache for one retained window plus one in-flight chunk."""
271-
assert window_size > 0, "Sliding-window size must be positive"
272-
if max_seq_len is None:
273-
max_seq_len = max_context_length
274-
assert max_seq_len > 0, "Maximum sequence length must be positive"
275-
assert window_size <= max_context_length, (
276-
f"Sliding-window size ({window_size}) cannot exceed the full context "
277-
f"length ({max_context_length})"
278-
)
279-
return min(max_context_length, window_size + max_seq_len)
280-
281-
282265
class RingKVCache(KVCache):
283266
def __init__(
284267
self,
@@ -288,17 +271,10 @@ def __init__(
288271
head_dim: int,
289272
enable_dynamic_shape: bool,
290273
dtype=torch.float32,
291-
*,
292-
window_size: int,
293-
max_seq_len: Optional[int] = None,
294274
):
275+
self.window_size = max_context_length
295276
"""
296-
The cache needs room for the retained sliding window and the current
297-
prefill chunk. Its size is window_size + max_seq_len, capped by the
298-
full-context cache. If max_seq_len is omitted, the full context length
299-
is used.
300-
301-
Reason why a cache larger than the sliding window is needed:
277+
Reason why we want the kv cache size to be twice the context length:
302278
Sliding window attention without ringbuffer
303279
pos 0 1 2 3 4 5 6 7 8 9 10
304280
0 x 0 0 0 0 0 0 0 0 0 0
@@ -331,28 +307,20 @@ def __init__(
331307
So not having kept 2, 3 and 4 in cache means we will have divergent behavior.
332308
Worst case of this would have been when update it equal to the length of
333309
the cache. like in our case pos = 5 seq len = 4.
334-
Thus we need a cache larger than the sliding window by enough space for
335-
the largest in-flight input chunk.
310+
Thus we need to have a cache that is larger. How much larger, as much as
311+
the sliding window size. So twice the max_context_length.
336312
How would that have helped. Lets see. At pos = 5 our cache would have
337313
[0, 1, 2, 3, 4, NA, NA, NA] After cache update we would have
338314
[8, 1, 2, 3, 4, 5, 6, 7]. We kicked out token at pos = 0. However, the
339315
current step still has access to [pos - sliding_window_size, pos] tokens.
340-
316+
341317
To make sure we dont over attend, i.e. we dont have pos = 5
342318
to attend to pos = 1, mask calculaton has to account for the sliding window
343319
size.
344320
"""
345-
self.window_size = window_size
346-
self.full_context_length = max_context_length
347-
self.max_seq_len = (
348-
max_context_length if max_seq_len is None else int(max_seq_len)
349-
)
350-
ring_cache_size = _get_ring_cache_size(
351-
max_context_length, window_size, self.max_seq_len
352-
)
353321
super().__init__(
354322
max_batch_size,
355-
ring_cache_size,
323+
max_context_length * 2,
356324
n_heads,
357325
head_dim,
358326
enable_dynamic_shape,

‎examples/models/llama/export_llama_lib.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -969,7 +969,6 @@ def _prepare_for_llama_export(llm_config: LlmConfig) -> LLMEdgeManager:
969969
preq_group_size=llm_config.base.preq_group_size,
970970
preq_embedding_quantize=llm_config.base.preq_embedding_quantize,
971971
local_global_attention=llm_config.model.local_global_attention,
972-
max_seq_len=llm_config.export.max_seq_length,
973972
use_torchao_kernels_linear=llm_config.backend.torchao.use_torchao_kernels_linear,
974973
use_torchao_kernels_tied_embedding=llm_config.backend.torchao.use_torchao_kernels_tied_embedding,
975974
quantize_with_hqq=llm_config.quantization.use_hqq,
@@ -2065,7 +2064,6 @@ def _get_source_transforms( # noqa
20652064
preq_group_size: Optional[int] = None,
20662065
preq_embedding_quantize: Optional[str] = None,
20672066
local_global_attention: Optional[List[int]] = None,
2068-
max_seq_len: Optional[int] = None,
20692067
use_torchao_kernels_linear: bool = False,
20702068
use_torchao_kernels_tied_embedding: bool = False,
20712069
quantize_with_hqq: bool = True,
@@ -2106,8 +2104,6 @@ def _get_source_transforms( # noqa
21062104
preq_mode: Pre-quantization mode.
21072105
preq_group_size: Pre-quantization group size.
21082106
preq_embedding_quantize: Pre-quantization embedding quantize.
2109-
max_seq_len: Largest input chunk accepted by local-attention layers. If
2110-
omitted, each layer's full context length is used.
21112107
21122108
Returns:
21132109
A list of transformation functions.
@@ -2284,7 +2280,6 @@ def _get_source_transforms( # noqa
22842280
partial(
22852281
replace_kv_cache_with_ring_kv_cache,
22862282
layer_sizes=local_global_attention,
2287-
max_seq_len=max_seq_len,
22882283
)
22892284
)
22902285

‎examples/models/llama/source_transformation/attention_sink.py‎

Lines changed: 37 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -18,36 +18,13 @@
1818
_create_causal_mask_for_ring_buffer,
1919
AttentionMHA,
2020
KVCache,
21+
RingKVCache,
2122
)
2223
from executorch.examples.models.llama.model_args import ModelArgs
2324
from executorch.examples.models.llama.rope import Rope
2425
from torchao.quantization.quant_api import _replace_with_custom_fn_if_matches_filter
2526

2627

27-
def _get_attention_sink_cache_size(
28-
max_context_length: int,
29-
window_size: int,
30-
sink_size: int,
31-
max_seq_len: Optional[int] = None,
32-
) -> int:
33-
"""Size a sink cache for fixed sinks, one window, and one input chunk."""
34-
assert sink_size >= 0, "Attention sink size must be non-negative"
35-
assert sink_size < max_context_length, (
36-
f"Attention sink size ({sink_size}) must be smaller than the full "
37-
f"context length ({max_context_length})"
38-
)
39-
assert window_size > 0, "Sliding-window size must be positive"
40-
assert sink_size + window_size <= max_context_length, (
41-
f"Attention sink size ({sink_size}) plus sliding-window size "
42-
f"({window_size}) cannot exceed the full context length "
43-
f"({max_context_length})"
44-
)
45-
if max_seq_len is None:
46-
max_seq_len = max_context_length
47-
assert max_seq_len > 0, "Maximum sequence length must be positive"
48-
return sink_size + window_size + max_seq_len
49-
50-
5128
class RopeWithAttentionSink(Rope):
5229
"""
5330
Rope subclass for Attention Sink models.
@@ -60,11 +37,10 @@ class RopeWithAttentionSink(Rope):
6037
- Window tokens (pos >= sink_size): wrapped into ring buffer range
6138
[sink_size, sink_size + ring_size) via modulo
6239
63-
The ring buffer holds one retained window plus one maximum-size input
64-
chunk. It is larger than the live window for write-ahead headroom, not to
65-
keep the live window contiguous -- it can span a wrap. Across a wrap two
66-
positions remap to a difference that is not their true distance, so RoPE
67-
preserves relative distance only within a wrap.
40+
The ring buffer is 2x window_size for write-ahead headroom, not to keep the
41+
live window contiguous -- it can span a wrap. Across a wrap two positions
42+
remap to a difference that is not their true distance, so RoPE preserves
43+
relative distance only within a wrap.
6844
"""
6945

7046
def __init__(
@@ -76,26 +52,7 @@ def __init__(
7652
super().__init__(params)
7753
self.window_size = window_size
7854
self.sink_size = sink_size
79-
max_seq_len = (
80-
params.max_context_len
81-
if getattr(params, "max_seq_len", None) is None
82-
else int(params.max_seq_len)
83-
)
84-
cache_size = _get_attention_sink_cache_size(
85-
params.max_context_len,
86-
window_size,
87-
sink_size,
88-
max_seq_len,
89-
)
90-
self.ring_size = cache_size - sink_size
91-
if self.freqs_cos.size(0) < cache_size:
92-
freqs_cos, freqs_sin = self.precompute_freqs_cis(
93-
self.params.head_dim,
94-
cache_size,
95-
self.params.rope_freq_base,
96-
)
97-
self.freqs_cos = freqs_cos
98-
self.freqs_sin = freqs_sin
55+
self.ring_size = window_size * 2
9956

10057
def _remap_input_pos(self, input_pos: torch.Tensor) -> torch.Tensor:
10158
"""Remap positions: sink tokens stay, window tokens wrap in ring buffer."""
@@ -176,7 +133,7 @@ class CachePositionsManagerWithSink(nn.Module):
176133
For sink_size=0: behaves exactly like original CachePositionsManager.
177134
For sink_size>0: sink tokens go to fixed positions, rest uses ring buffer.
178135
179-
IMPORTANT: cache_size is the actual cache dimension, including sink slots.
136+
IMPORTANT: cache_size should be the actual cache dimension size (2x window for ring buffer).
180137
"""
181138

182139
def __init__(self, cache_size: int, sink_size: int = 0):
@@ -234,7 +191,7 @@ class KVCacheWithAttentionSink(KVCache):
234191
Uses a ring buffer approach for the sliding window portion while keeping
235192
the first sink_size tokens fixed. This avoids dynamic shape operations.
236193
237-
Cache layout: [fixed sink tokens] [ring buffer for one window + one chunk]
194+
Cache layout: [sink: 0 to sink_size-1] [ring_buffer: sink_size to sink_size + window_size*2 - 1]
238195
"""
239196

240197
def __init__(
@@ -245,27 +202,16 @@ def __init__(
245202
rope: RopeWithAttentionSink,
246203
window_size: int,
247204
sink_size: int,
248-
max_context_length: int,
249-
max_seq_len: Optional[int] = None,
250205
max_batch_size: int = 1,
251206
dtype=torch.float32,
252207
):
253-
self.full_context_length = max_context_length
254-
self.max_seq_len = (
255-
max_context_length if max_seq_len is None else int(max_seq_len)
256-
)
257-
# Keep the fixed sinks separate from the ring space needed for the
258-
# retained window and the largest in-flight input chunk.
259-
total_cache_size = _get_attention_sink_cache_size(
260-
max_context_length,
261-
window_size,
262-
sink_size,
263-
self.max_seq_len,
264-
)
265-
assert rope.ring_size == total_cache_size - sink_size, (
266-
f"RoPE ring size ({rope.ring_size}) must match the KV-cache ring "
267-
f"size ({total_cache_size - sink_size})"
268-
)
208+
# Total cache size is sink_size + window_size * 2.
209+
# The ring buffer needs 2x the window size because at the moment a new
210+
# token is written, the previous window_size tokens must still be readable
211+
# (they haven't been overwritten yet). With only 1x, writing a new entry
212+
# would immediately evict the oldest visible token, leaving fewer than
213+
# window_size tokens available for attention.
214+
total_cache_size = sink_size + window_size * 2
269215
super().__init__(
270216
max_batch_size=max_batch_size,
271217
max_context_length=total_cache_size,
@@ -364,7 +310,6 @@ def _replace_attention(
364310
rope_with_attention_sink: RopeWithAttentionSink,
365311
sink_size: int,
366312
window_size: int,
367-
max_seq_len: Optional[int],
368313
):
369314
for _, child_module in module._modules.items():
370315
if len(list(child_module.children())) > 0: # pyre-ignore [16]
@@ -373,23 +318,32 @@ def _replace_attention(
373318
rope_with_attention_sink=rope_with_attention_sink,
374319
sink_size=sink_size,
375320
window_size=window_size,
376-
max_seq_len=max_seq_len,
377321
)
378322

379323
if isinstance(child_module, AttentionMHA):
380324
kv_cache = child_module.kv_cache
381-
child_module.kv_cache = KVCacheWithAttentionSink(
382-
n_heads=kv_cache.n_heads,
383-
head_dim=kv_cache.head_dim,
384-
enable_dynamic_shape=kv_cache.enable_dynamic_shape,
385-
rope=rope_with_attention_sink,
386-
max_batch_size=kv_cache.max_batch_size,
387-
window_size=window_size,
388-
sink_size=sink_size,
389-
max_context_length=kv_cache.max_context_length,
390-
max_seq_len=max_seq_len,
391-
dtype=kv_cache.k_cache.dtype,
392-
)
325+
if sink_size == 0:
326+
# No sink tokens needed — use standard RingKVCache directly
327+
child_module.kv_cache = RingKVCache(
328+
kv_cache.max_batch_size,
329+
window_size, # RingKVCache expects user-provided window size
330+
kv_cache.n_heads,
331+
kv_cache.head_dim,
332+
kv_cache.enable_dynamic_shape,
333+
kv_cache.k_cache.dtype,
334+
)
335+
else:
336+
kv_cache_with_attention_sink = KVCacheWithAttentionSink(
337+
n_heads=kv_cache.n_heads,
338+
head_dim=kv_cache.head_dim,
339+
enable_dynamic_shape=kv_cache.enable_dynamic_shape,
340+
rope=rope_with_attention_sink,
341+
max_batch_size=kv_cache.max_batch_size,
342+
window_size=window_size,
343+
sink_size=sink_size,
344+
dtype=kv_cache.k_cache.dtype,
345+
)
346+
child_module.kv_cache = kv_cache_with_attention_sink
393347
# Don't replace forward - let the original AttentionMHA.forward handle it
394348
# since our KVCache has is_ring_buffer=True, it will use the ring buffer mask
395349

@@ -411,13 +365,11 @@ def enable_attention_sink(
411365
window_size=window_size,
412366
sink_size=sink_size,
413367
)
414-
max_seq_len = getattr(params, "max_seq_len", None)
415368
_replace_rope(module, rope_with_attention_sink)
416369
_replace_attention(
417370
module=module,
418371
rope_with_attention_sink=rope_with_attention_sink,
419372
sink_size=sink_size,
420373
window_size=window_size,
421-
max_seq_len=max_seq_len,
422374
)
423375
return module

0 commit comments

Comments
 (0)