Skip to content

Commit 138b69d

Browse files
authored
Arm backend: Add test for StaticCache in FP32 (pytorch#17334)
* Adds workaround for graphs with mutable buffers in the tester * Add test for HF StaticCache Change-Id: I17fa52579031bed4f0883d0ccf6f16f0653a2ade cc @freddan80 @per @zingo @oscarandersson8218 @digantdesai --------- Signed-off-by: Tom Allsop <tom.allsop@arm.com>
1 parent bc469e7 commit 138b69d

2 files changed

Lines changed: 211 additions & 1 deletion

File tree

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
# Copyright 2026 Arm Limited and/or its affiliates.
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
from typing import Tuple
7+
8+
import pytest
9+
10+
import torch
11+
from executorch.backends.arm._passes.insert_int32_casts_after_int64_placeholders import (
12+
InsertInt32CastsAfterInt64PlaceholdersPass,
13+
)
14+
from executorch.backends.arm.test import common
15+
from executorch.backends.arm.test.tester.test_pipeline import (
16+
EthosU55PipelineINT,
17+
EthosU85PipelineINT,
18+
TosaPipelineFP,
19+
TosaPipelineINT,
20+
VgfPipeline,
21+
)
22+
from transformers import LlamaConfig
23+
from transformers.cache_utils import StaticCache
24+
25+
input_t = Tuple[torch.Tensor, torch.Tensor, torch.Tensor]
26+
27+
28+
test_configs = {
29+
"multihead_attention": LlamaConfig(num_attention_heads=32),
30+
"grouped_query_attention": LlamaConfig(
31+
num_attention_heads=32, num_key_value_heads=4
32+
),
33+
"multi_query_attention": LlamaConfig(num_attention_heads=32, num_key_value_heads=1),
34+
}
35+
36+
37+
@torch.no_grad()
38+
class StaticCacheModule(torch.nn.Module):
39+
def __init__(
40+
self,
41+
config: LlamaConfig,
42+
max_cache_len: int = 10,
43+
dtype: torch.dtype = torch.float32,
44+
) -> None:
45+
super().__init__()
46+
47+
assert dtype is torch.float32
48+
49+
self.config = config
50+
self.dtype = dtype
51+
52+
self.cache = StaticCache(config=self.config, max_cache_len=max_cache_len)
53+
54+
hidden_size = self.config.hidden_size
55+
num_attention_heads = self.config.num_attention_heads
56+
assert hidden_size is not None and num_attention_heads is not None
57+
58+
self.hidden_size = hidden_size
59+
self.num_attention_heads = num_attention_heads
60+
61+
self.cache.early_initialization(
62+
1,
63+
self.num_attention_heads,
64+
self.hidden_size // self.num_attention_heads,
65+
self.dtype,
66+
torch.device("cpu"),
67+
)
68+
69+
for i in range(len(self.cache.layers)):
70+
self.register_buffer(f"cache_layer_keys_{i}", self.cache.layers[i].keys)
71+
self.register_buffer(f"cache_layer_values_{i}", self.cache.layers[i].values)
72+
73+
def forward(
74+
self,
75+
key_states: torch.Tensor,
76+
value_states: torch.Tensor,
77+
cache_position: torch.Tensor,
78+
) -> Tuple[torch.Tensor, torch.Tensor]:
79+
key, value = self.cache.update(
80+
key_states,
81+
value_states,
82+
layer_idx=0,
83+
cache_kwargs={
84+
"cache_position": cache_position,
85+
},
86+
)
87+
88+
return key.clone(), value.clone()
89+
90+
def get_inputs(self) -> input_t:
91+
key_states = torch.randn(
92+
(
93+
1,
94+
self.num_attention_heads,
95+
1,
96+
self.hidden_size // self.num_attention_heads,
97+
),
98+
dtype=torch.float32,
99+
)
100+
value_states = torch.randn(
101+
(
102+
1,
103+
self.num_attention_heads,
104+
1,
105+
self.hidden_size // self.num_attention_heads,
106+
),
107+
dtype=torch.float32,
108+
)
109+
cache_position = torch.tensor([1], dtype=torch.int64)
110+
111+
return key_states, value_states, cache_position
112+
113+
114+
@common.parametrize("test_data", test_configs)
115+
def test_static_cache_tosa_FP(test_data):
116+
module = StaticCacheModule(test_data).eval()
117+
pipeline = TosaPipelineFP[input_t](
118+
module,
119+
module.get_inputs(),
120+
aten_op=[],
121+
exir_op=[],
122+
transform_passes=[InsertInt32CastsAfterInt64PlaceholdersPass()],
123+
)
124+
pipeline.run()
125+
126+
127+
@pytest.mark.xfail(
128+
reason="TODO(MLETORCH-1818): Quantization for StaticCache is not yet supported."
129+
)
130+
@common.parametrize("test_data", test_configs)
131+
def test_static_cache_tosa_INT(test_data):
132+
module = StaticCacheModule(test_data).eval()
133+
pipeline = TosaPipelineINT[input_t](
134+
module,
135+
module.get_inputs(),
136+
aten_op=[],
137+
exir_op=[],
138+
)
139+
pipeline.run()
140+
141+
142+
@common.XfailIfNoCorstone300
143+
@pytest.mark.xfail(
144+
reason="Quantization for StaticCache is not yet supported. Scatter operator is also not supported on U55."
145+
)
146+
@common.parametrize("test_data", test_configs)
147+
def test_static_cache_u55_INT(test_data):
148+
module = StaticCacheModule(test_data).eval()
149+
pipeline = EthosU55PipelineINT[input_t](
150+
module,
151+
module.get_inputs(),
152+
aten_ops=[],
153+
)
154+
pipeline.run()
155+
156+
157+
@common.XfailIfNoCorstone320
158+
@pytest.mark.xfail(
159+
reason="TODO(MLETORCH-1818): Quantization for StaticCache is not yet supported."
160+
)
161+
@common.parametrize("test_data", test_configs)
162+
def test_static_cache_u85_INT(test_data):
163+
module = StaticCacheModule(test_data).eval()
164+
pipeline = EthosU85PipelineINT[input_t](module, module.get_inputs(), aten_ops=[])
165+
pipeline.run()
166+
167+
168+
@common.SkipIfNoModelConverter
169+
@common.parametrize("test_data", test_configs)
170+
def test_static_cache_vgf_no_quant(test_data):
171+
module = StaticCacheModule(test_data).eval()
172+
pipeline = VgfPipeline[input_t](
173+
module,
174+
module.get_inputs(),
175+
aten_op=[],
176+
exir_op=[],
177+
transform_passes=[InsertInt32CastsAfterInt64PlaceholdersPass()],
178+
quantize=False,
179+
)
180+
pipeline.run()
181+
182+
183+
@common.SkipIfNoModelConverter
184+
@pytest.mark.xfail(
185+
reason="TODO(MLETORCH-1818): Quantization for StaticCache is not yet supported."
186+
)
187+
@common.parametrize("test_data", test_configs)
188+
def test_static_cache_vgf_quant(test_data):
189+
module = StaticCacheModule(test_data).eval()
190+
pipeline = VgfPipeline[input_t](
191+
module,
192+
module.get_inputs(),
193+
aten_op=[],
194+
exir_op=[],
195+
quantize=True,
196+
)
197+
pipeline.run()

‎backends/arm/test/tester/arm_tester.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,20 @@ def run(
198198
class ToExecutorch(tester.ToExecutorch):
199199
def run_artifact(self, inputs):
200200
with TosaReferenceModelDispatch():
201-
return super().run_artifact(inputs)
201+
# Check if the model has mutable buffers. These are not delegated to the backend
202+
# and are handled by core ExecuTorch as I/O. In other words, the mutable buffer
203+
# is outputted and re-inputted into the model. As we are calling the graph module
204+
# directly, we need to ensure we handle these extra mutable inputs.
205+
if (
206+
len(self.artifact.exported_program().graph_signature.buffers_to_mutate)
207+
> 0
208+
):
209+
buffers = list(self.artifact.exported_program().buffers())
210+
buffers.extend(inputs)
211+
212+
return self.artifact.exported_program().graph_module(*buffers)
213+
else:
214+
return super().run_artifact(inputs)
202215

203216

204217
class RunPasses(tester.RunPasses):

0 commit comments

Comments
 (0)