@@ -64,61 +64,55 @@ def apply_update(
6464 """
6565 After inference, update the cache state for next iteration. The runtime needs to
6666 implement the same operation.
67+
68+ `pos` is the absolute position of the first token in the update. Caches smaller
69+ than the sequence (sliding-window layers) wrap it into a ring index.
6770 """
6871 seq_dim = - 1 if transpose else - 2
6972 cache_len = cache .size (seq_dim )
7073 if cache_len == 0 :
7174 return
72- if cache_len < update .size (seq_dim ):
73- update = torch .narrow (
74- update ,
75- seq_dim ,
76- update .size (seq_dim ) - cache_len ,
77- cache_len ,
78- )
79- assert update .size (seq_dim ) == cache_len
75+ if update_len is None :
76+ update_len = update .size (seq_dim )
77+
78+ # Only the newest cache_len tokens fit. Drop from the front of the real-token
79+ # window instead of narrowing the update tensor, which may be padded past
80+ # update_len and would otherwise push padding into the cache.
81+ dropped = max (0 , update_len - cache_len )
82+ update_pos += dropped
83+ update_len -= dropped
84+ pos = (pos + dropped ) % cache_len
8085
8186 if style == "shift_pointer" :
87+ updated = torch .roll (cache , - update_len , seq_dim )
8288 if transpose :
83- update_len = update_len or update .size (- 1 )
84- updated = torch .roll (cache , - update_len , - 1 )
8589 updated [..., - update_len :] = update [
8690 ..., update_pos : update_pos + update_len
8791 ]
8892 else :
89- update_len = update_len or update .size (- 2 )
90- updated = torch .roll (cache , - update_len , - 2 )
9193 updated [..., - update_len :, :] = update [
9294 ..., update_pos : update_pos + update_len , :
9395 ]
9496
9597 if style == "smart_mask" :
96- available = cache .size (- 2 ) - pos
97- update_len = update_len or update .size (- 1 if transpose else - 2 )
98- if update_len > available :
99- wrap = update_len - available
100- update_len = available
101- else :
102- wrap = 0
98+ # Ring buffer: fill to the end of the cache, then wrap to the front.
99+ contiguous_len = min (update_len , cache_len - pos )
100+ wrap = update_len - contiguous_len
101+ wrap_pos = update_pos + contiguous_len
103102
104103 updated = torch .clone (cache )
105104 if transpose :
106- updated [..., pos : pos + update_len ] = update [
107- ..., update_pos : update_pos + update_len
105+ updated [..., pos : pos + contiguous_len ] = update [
106+ ..., update_pos : update_pos + contiguous_len
108107 ]
109108 if wrap > 0 :
110- update_pos += update_len
111- updated [..., :wrap ] = update [..., update_pos : update_pos + wrap ]
112-
109+ updated [..., :wrap ] = update [..., wrap_pos : wrap_pos + wrap ]
113110 else :
114- updated [..., pos : pos + update_len , :] = update [
115- ..., update_pos : update_pos + update_len , :
111+ updated [..., pos : pos + contiguous_len , :] = update [
112+ ..., update_pos : update_pos + contiguous_len , :
116113 ]
117114 if wrap > 0 :
118- update_pos += update_len
119- updated [..., :wrap , :] = update [
120- ..., update_pos : update_pos + wrap , :
121- ]
115+ updated [..., :wrap , :] = update [..., wrap_pos : wrap_pos + wrap , :]
122116
123117 return updated
124118
@@ -198,16 +192,21 @@ def set_input_mask(self, input_mask):
198192 self .tensor [:, :, self .cache_len :] = input_mask
199193
200194 def unmask (self , new_unmasked_len ):
195+ # Clamp to what is left of the cache region, mirroring the runtime
196+ # (static_attention_io_manager.h). Without this the smart_mask branch walks
197+ # past cache_len and zeroes part of the in-chunk causal mask.
198+ new_unmasked_len = min (new_unmasked_len , self .cache_len - self .unmasked_len )
201199 if new_unmasked_len <= 0 :
202200 return
203201
204202 if self .style == "shift_pointer" :
205203 self .tensor [
206204 :,
207205 :,
208- max (0 , self .cache_len - self .unmasked_len - new_unmasked_len ) : max (
209- 0 , self .cache_len - self .unmasked_len
210- ),
206+ self .cache_len
207+ - self .unmasked_len
208+ - new_unmasked_len : self .cache_len
209+ - self .unmasked_len ,
211210 ] = 0
212211
213212 if self .style == "smart_mask" :
@@ -440,24 +439,34 @@ def _mask_cache_outside_window(self):
440439 its own window. `unmask` fills the cache region identically for all rows,
441440 which lets query row k attend to cache_len + k + 1 keys instead of cache_len.
442441
443- With shift_pointer the valid cache entries are right aligned and ordered
444- oldest to newest, so with W = cache_len and v = min(pos, W) valid entries,
445- cache column c is visible to query row k iff c >= max(W - v, k + 1). This
446- holds for both prefill chunks and decode steps, and composes with the
447- in-chunk causal/window mask set by the caller.
442+ Express both layouts through the age of the token held in a slot -- how many
443+ tokens are newer than it. With W = cache_len and v = min(pos, W) valid entries,
444+ slot c is visible to query row k iff age(c) < min(v, W - 1 - k): the first term
445+ is validity, the second is the window. Once k >= W - 1 the whole window lives
446+ inside the chunk and nothing in the cache is visible.
447+
448+ Only the age mapping differs by layout. shift_pointer keeps valid entries right
449+ aligned oldest to newest, so age is W - 1 - c. smart_mask is a ring indexed by
450+ absolute position, so the same pattern is rotated by pos.
451+
452+ Composes with the in-chunk causal/window mask set by the caller, and holds for
453+ both prefill chunks and decode steps.
448454 """
449455 for mask in self ._masks .values ():
450456 if not self ._is_windowed_mask (mask ):
451457 continue
452458 w = mask .cache_len
453- first_valid = w - min (self .pos , w )
454459 rows = torch .arange (self .input_len ).unsqueeze (1 )
455460 cols = torch .arange (w ).unsqueeze (0 )
456- visible = cols >= torch .clamp (rows + 1 , min = first_valid )
461+ if mask .style == "smart_mask" :
462+ age = (self .pos - 1 - cols ) % w
463+ else :
464+ age = w - 1 - cols
465+ limit = torch .clamp (w - 1 - rows , max = min (self .pos , w ))
457466 cache_mask = torch .full (
458467 (self .input_len , w ), mask .mask_val , dtype = mask .tensor .dtype
459468 )
460- cache_mask [visible ] = 0
469+ cache_mask [age < limit ] = 0
461470 mask .tensor [:, :, :w ] = cache_mask
462471
463472 def prefill (
0 commit comments