Skip to content

Commit 1d4df21

Browse files
authored
Support sliding-window layers on the smart_mask cache ring (pytorch#21929)
Differential Revision: D116524534 Pull Request resolved: pytorch#21929
1 parent 15495a5 commit 1d4df21

2 files changed

Lines changed: 97 additions & 49 deletions

File tree

‎examples/models/llama/static_attention.py‎

Lines changed: 50 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -64,61 +64,55 @@ def apply_update(
6464
"""
6565
After inference, update the cache state for next iteration. The runtime needs to
6666
implement the same operation.
67+
68+
`pos` is the absolute position of the first token in the update. Caches smaller
69+
than the sequence (sliding-window layers) wrap it into a ring index.
6770
"""
6871
seq_dim = -1 if transpose else -2
6972
cache_len = cache.size(seq_dim)
7073
if cache_len == 0:
7174
return
72-
if cache_len < update.size(seq_dim):
73-
update = torch.narrow(
74-
update,
75-
seq_dim,
76-
update.size(seq_dim) - cache_len,
77-
cache_len,
78-
)
79-
assert update.size(seq_dim) == cache_len
75+
if update_len is None:
76+
update_len = update.size(seq_dim)
77+
78+
# Only the newest cache_len tokens fit. Drop from the front of the real-token
79+
# window instead of narrowing the update tensor, which may be padded past
80+
# update_len and would otherwise push padding into the cache.
81+
dropped = max(0, update_len - cache_len)
82+
update_pos += dropped
83+
update_len -= dropped
84+
pos = (pos + dropped) % cache_len
8085

8186
if style == "shift_pointer":
87+
updated = torch.roll(cache, -update_len, seq_dim)
8288
if transpose:
83-
update_len = update_len or update.size(-1)
84-
updated = torch.roll(cache, -update_len, -1)
8589
updated[..., -update_len:] = update[
8690
..., update_pos : update_pos + update_len
8791
]
8892
else:
89-
update_len = update_len or update.size(-2)
90-
updated = torch.roll(cache, -update_len, -2)
9193
updated[..., -update_len:, :] = update[
9294
..., update_pos : update_pos + update_len, :
9395
]
9496

9597
if style == "smart_mask":
96-
available = cache.size(-2) - pos
97-
update_len = update_len or update.size(-1 if transpose else -2)
98-
if update_len > available:
99-
wrap = update_len - available
100-
update_len = available
101-
else:
102-
wrap = 0
98+
# Ring buffer: fill to the end of the cache, then wrap to the front.
99+
contiguous_len = min(update_len, cache_len - pos)
100+
wrap = update_len - contiguous_len
101+
wrap_pos = update_pos + contiguous_len
103102

104103
updated = torch.clone(cache)
105104
if transpose:
106-
updated[..., pos : pos + update_len] = update[
107-
..., update_pos : update_pos + update_len
105+
updated[..., pos : pos + contiguous_len] = update[
106+
..., update_pos : update_pos + contiguous_len
108107
]
109108
if wrap > 0:
110-
update_pos += update_len
111-
updated[..., :wrap] = update[..., update_pos : update_pos + wrap]
112-
109+
updated[..., :wrap] = update[..., wrap_pos : wrap_pos + wrap]
113110
else:
114-
updated[..., pos : pos + update_len, :] = update[
115-
..., update_pos : update_pos + update_len, :
111+
updated[..., pos : pos + contiguous_len, :] = update[
112+
..., update_pos : update_pos + contiguous_len, :
116113
]
117114
if wrap > 0:
118-
update_pos += update_len
119-
updated[..., :wrap, :] = update[
120-
..., update_pos : update_pos + wrap, :
121-
]
115+
updated[..., :wrap, :] = update[..., wrap_pos : wrap_pos + wrap, :]
122116

123117
return updated
124118

@@ -198,16 +192,21 @@ def set_input_mask(self, input_mask):
198192
self.tensor[:, :, self.cache_len :] = input_mask
199193

200194
def unmask(self, new_unmasked_len):
195+
# Clamp to what is left of the cache region, mirroring the runtime
196+
# (static_attention_io_manager.h). Without this the smart_mask branch walks
197+
# past cache_len and zeroes part of the in-chunk causal mask.
198+
new_unmasked_len = min(new_unmasked_len, self.cache_len - self.unmasked_len)
201199
if new_unmasked_len <= 0:
202200
return
203201

204202
if self.style == "shift_pointer":
205203
self.tensor[
206204
:,
207205
:,
208-
max(0, self.cache_len - self.unmasked_len - new_unmasked_len) : max(
209-
0, self.cache_len - self.unmasked_len
210-
),
206+
self.cache_len
207+
- self.unmasked_len
208+
- new_unmasked_len : self.cache_len
209+
- self.unmasked_len,
211210
] = 0
212211

213212
if self.style == "smart_mask":
@@ -440,24 +439,34 @@ def _mask_cache_outside_window(self):
440439
its own window. `unmask` fills the cache region identically for all rows,
441440
which lets query row k attend to cache_len + k + 1 keys instead of cache_len.
442441
443-
With shift_pointer the valid cache entries are right aligned and ordered
444-
oldest to newest, so with W = cache_len and v = min(pos, W) valid entries,
445-
cache column c is visible to query row k iff c >= max(W - v, k + 1). This
446-
holds for both prefill chunks and decode steps, and composes with the
447-
in-chunk causal/window mask set by the caller.
442+
Express both layouts through the age of the token held in a slot -- how many
443+
tokens are newer than it. With W = cache_len and v = min(pos, W) valid entries,
444+
slot c is visible to query row k iff age(c) < min(v, W - 1 - k): the first term
445+
is validity, the second is the window. Once k >= W - 1 the whole window lives
446+
inside the chunk and nothing in the cache is visible.
447+
448+
Only the age mapping differs by layout. shift_pointer keeps valid entries right
449+
aligned oldest to newest, so age is W - 1 - c. smart_mask is a ring indexed by
450+
absolute position, so the same pattern is rotated by pos.
451+
452+
Composes with the in-chunk causal/window mask set by the caller, and holds for
453+
both prefill chunks and decode steps.
448454
"""
449455
for mask in self._masks.values():
450456
if not self._is_windowed_mask(mask):
451457
continue
452458
w = mask.cache_len
453-
first_valid = w - min(self.pos, w)
454459
rows = torch.arange(self.input_len).unsqueeze(1)
455460
cols = torch.arange(w).unsqueeze(0)
456-
visible = cols >= torch.clamp(rows + 1, min=first_valid)
461+
if mask.style == "smart_mask":
462+
age = (self.pos - 1 - cols) % w
463+
else:
464+
age = w - 1 - cols
465+
limit = torch.clamp(w - 1 - rows, max=min(self.pos, w))
457466
cache_mask = torch.full(
458467
(self.input_len, w), mask.mask_val, dtype=mask.tensor.dtype
459468
)
460-
cache_mask[visible] = 0
469+
cache_mask[age < limit] = 0
461470
mask.tensor[:, :, :w] = cache_mask
462471

463472
def prefill(

‎examples/models/llama/tests/test_static_attention.py‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ def sliding_window_mask(window):
269269
diagonal=-(window - 1),
270270
)
271271

272-
def test(window, chunk_len):
272+
def test(window, chunk_len, style):
273273
config = ModelArgs(
274274
dim=64,
275275
n_heads=4,
@@ -292,18 +292,57 @@ def test(window, chunk_len):
292292

293293
cache_lens = [window, global_cache_len]
294294
single_chunk = StaticAttentionIOManager(
295-
static_config, prompt_len, cache_lens
295+
static_config, prompt_len, cache_lens, style=style
296296
).prefill(static_transformer, x)
297297
chunked = StaticAttentionIOManager(
298-
static_config, chunk_len, cache_lens
298+
static_config, chunk_len, cache_lens, style=style
299299
).prefill(static_transformer, x)
300300

301-
msg = f"window={window}, chunk_len={chunk_len}"
302-
self.assertTrue(torch.isclose(chunked, single_chunk, rtol=1e-3).all(), msg)
303-
self.assertTrue(torch.isclose(chunked, expected, rtol=1e-3).all(), msg)
301+
msg = f"window={window}, chunk_len={chunk_len}, style={style}"
302+
# atol covers logits that land near zero, where a pure relative tolerance
303+
# is meaningless -- float noise here is ~1e-7 while a genuine windowing
304+
# regression moves logits by ~1e-1.
305+
self.assertTrue(
306+
torch.isclose(chunked, single_chunk, rtol=1e-3, atol=1e-5).all(), msg
307+
)
308+
self.assertTrue(
309+
torch.isclose(chunked, expected, rtol=1e-3, atol=1e-5).all(), msg
310+
)
311+
312+
for style in ("shift_pointer", "smart_mask"):
313+
test(window=4, chunk_len=8, style=style) # chunk longer than the window
314+
test(window=12, chunk_len=6, style=style) # chunk shorter than the window
315+
316+
def test_sliding_window_cache_ring(self):
317+
"""A cache smaller than the sequence keeps the newest cache_len tokens.
318+
319+
`pos` is the absolute position of the update, so it exceeds cache_len as soon
320+
as the window has been filled once, and an update can be longer than the cache.
321+
"""
322+
323+
def test(style, chunk_len):
324+
cache_len = 8
325+
n_chunks = 5
326+
cache = torch.full((1, cache_len, 1), -1, dtype=torch.int64)
327+
for i in range(n_chunks):
328+
pos = i * chunk_len
329+
tokens = torch.arange(pos, pos + chunk_len, dtype=torch.int64).view(
330+
1, chunk_len, 1
331+
)
332+
cache = StaticKVCache.apply_update(cache, tokens, pos, style)
333+
334+
total = n_chunks * chunk_len
335+
expected = set(range(max(0, total - cache_len), total))
336+
self.assertEqual(
337+
set(cache.flatten().tolist()),
338+
expected,
339+
f"style={style}, chunk_len={chunk_len}",
340+
)
304341

305-
test(window=4, chunk_len=8) # chunk longer than the window
306-
test(window=12, chunk_len=6) # chunk shorter than the window
342+
for style in ("shift_pointer", "smart_mask"):
343+
test(style, chunk_len=2) # update shorter than the cache
344+
test(style, chunk_len=8) # update exactly the cache length
345+
test(style, chunk_len=12) # update longer than the cache
307346

308347
def test_lookahead_decode(self):
309348
config = ModelArgs(

0 commit comments

Comments
 (0)