Skip to content

Commit 3dad019

Browse files
Arm backend: Add tests for Stable Diffusion 3.5 Large model (pytorch#22606)
- Add SD3.5 Large test configs for CLIP text encoders, T5 text encoder, MMDiT transformer, and VAE decoder, with optional upstream-sync validation. - Add Arm backend lowering tests for CLIP-L, CLIP-bigG, T5, transformer, and VAE decoder components across TOSA and VGF pipelines. - Add wrapper and model-loader unit tests covering wrapper output signatures, component subfolder loading, selective component loading, and dummy input shape generation. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani --------- Signed-off-by: Sangwon Ha <sangwon.ha@arm.com>
1 parent e1e551b commit 3dad019

6 files changed

Lines changed: 1397 additions & 0 deletions
Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
# Copyright 2026 Arm Limited and/or its affiliates.
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
from typing import Tuple
7+
8+
import pytest
9+
import torch
10+
from executorch.backends.arm.test import common
11+
from executorch.backends.arm.test.models.stable_diffusion_3_5_large.test_configs_sd35_large import (
12+
get_int64_to_int32_passes,
13+
get_tiny_sd35_large_text_encoder_2_config,
14+
get_tiny_sd35_large_text_encoder_config,
15+
)
16+
from executorch.backends.arm.test.tester.test_pipeline import (
17+
TosaPipelineFP,
18+
TosaPipelineINT,
19+
VgfPipeline,
20+
)
21+
from executorch.examples.models.stable_diffusion_3_5_large.model import (
22+
SD3CLIPTextEncoderWrapper,
23+
)
24+
from transformers import CLIPTextModelWithProjection
25+
26+
input_t = Tuple[torch.Tensor]
27+
28+
29+
class TestCLIPTextModelWithProjection:
30+
"""Test helper for SD3.5 Large CLIPTextModelWithProjection configs."""
31+
32+
ops_after_partitioner_FP = {
33+
"executorch_exir_dialects_edge__ops_aten_argmax_default": 1,
34+
"executorch_exir_dialects_edge__ops_aten_slice_copy_Tensor": 1,
35+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default": 2,
36+
"torch.ops.higher_order.executorch_call_delegate": 2,
37+
}
38+
39+
ops_after_partitioner_vgf_no_quantize = {
40+
"executorch_exir_dialects_edge__ops_aten_argmax_default": 1,
41+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default": 2,
42+
"torch.ops.higher_order.executorch_call_delegate": 2,
43+
}
44+
ops_after_partitioner_vgf_quantize = {
45+
"executorch_exir_dialects_edge__ops_aten_argmax_default": 1,
46+
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
47+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default": 2,
48+
"torch.ops.higher_order.executorch_call_delegate": 1,
49+
}
50+
51+
def create_dummy_inputs(
52+
self,
53+
config,
54+
batch_size: int = 1,
55+
seq_length: int = 2,
56+
dtype: torch.dtype = torch.long,
57+
) -> tuple[torch.Tensor]:
58+
"""Create dummy inputs for the CLIPTextModelWithProjection tests."""
59+
# SD3.5 Large uses (batch_size, seq_length) = (1, 77) for both CLIP-L
60+
# and CLIP-bigG. Keep this unit-test default smaller for TOSA runtime.
61+
return (
62+
torch.randint(
63+
low=0,
64+
high=config.vocab_size,
65+
size=(batch_size, seq_length),
66+
dtype=dtype,
67+
),
68+
)
69+
70+
def create_model(
71+
self,
72+
config,
73+
) -> SD3CLIPTextEncoderWrapper:
74+
"""Instantiate wrapped CLIPTextModelWithProjection for tests."""
75+
return SD3CLIPTextEncoderWrapper(
76+
CLIPTextModelWithProjection(config).to(dtype=config.dtype) # type: ignore[call-arg]
77+
).eval()
78+
79+
@staticmethod
80+
def ops_after_partitioner_INT(config) -> dict[str, int]:
81+
if config.num_hidden_layers == 2:
82+
return {
83+
"executorch_exir_dialects_edge__ops_aten_add_Tensor": 2,
84+
"executorch_exir_dialects_edge__ops_aten_argmax_default": 1,
85+
"executorch_exir_dialects_edge__ops_aten_where_self": 2,
86+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default": 12,
87+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 18,
88+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 12,
89+
"torch.ops.higher_order.executorch_call_delegate": 7,
90+
}
91+
92+
raise ValueError(
93+
f"Unexpected CLIP config: hidden_act={config.hidden_act}, "
94+
f"num_hidden_layers={config.num_hidden_layers}"
95+
)
96+
97+
98+
@pytest.mark.parametrize(
99+
("config_factory", "atol"),
100+
(
101+
(get_tiny_sd35_large_text_encoder_config, 1e-2), # FP atol
102+
(get_tiny_sd35_large_text_encoder_2_config, 1.5e-2), # FP atol
103+
),
104+
ids=("text_encoder", "text_encoder_2"),
105+
)
106+
def test_clip_text_model_with_projection_tosa_FP(config_factory, atol):
107+
"""Run the CLIPTextModelWithProjection TOSA FP test for a given config."""
108+
test_helper = TestCLIPTextModelWithProjection()
109+
config = config_factory()
110+
111+
with torch.no_grad():
112+
pipeline = TosaPipelineFP[input_t](
113+
test_helper.create_model(config),
114+
test_helper.create_dummy_inputs(config),
115+
aten_op=[],
116+
exir_op=[],
117+
use_to_edge_transform_and_lower=True,
118+
atol=atol,
119+
transform_passes=get_int64_to_int32_passes(),
120+
)
121+
pipeline.change_args(
122+
"check_count.exir", TestCLIPTextModelWithProjection.ops_after_partitioner_FP
123+
)
124+
pipeline.run()
125+
126+
127+
@pytest.mark.parametrize(
128+
("config_factory", "atol"),
129+
(
130+
(get_tiny_sd35_large_text_encoder_config, 5.5e-2), # INT atol
131+
(get_tiny_sd35_large_text_encoder_2_config, 6e-2), # INT atol
132+
),
133+
ids=("text_encoder", "text_encoder_2"),
134+
)
135+
def test_clip_text_model_with_projection_tosa_INT(config_factory, atol):
136+
"""Run the CLIPTextModelWithProjection TOSA INT test for a given config."""
137+
test_helper = TestCLIPTextModelWithProjection()
138+
config = config_factory()
139+
140+
with torch.no_grad():
141+
pipeline = TosaPipelineINT[input_t](
142+
test_helper.create_model(config),
143+
test_helper.create_dummy_inputs(config),
144+
aten_op=[],
145+
exir_op=[],
146+
use_to_edge_transform_and_lower=True,
147+
atol=atol,
148+
frobenius_threshold=None,
149+
cosine_threshold=None,
150+
)
151+
pipeline.change_args(
152+
"check_count.exir",
153+
TestCLIPTextModelWithProjection.ops_after_partitioner_INT(config),
154+
)
155+
pipeline.run()
156+
157+
158+
@common.SkipIfNoModelConverter
159+
@pytest.mark.parametrize(
160+
("config_factory",),
161+
(
162+
(get_tiny_sd35_large_text_encoder_config,),
163+
(get_tiny_sd35_large_text_encoder_2_config,),
164+
),
165+
ids=("text_encoder", "text_encoder_2"),
166+
)
167+
def test_clip_text_model_with_projection_vgf_no_quant(config_factory):
168+
"""Run the CLIPTextModelWithProjection VGF no-quant test."""
169+
test_helper = TestCLIPTextModelWithProjection()
170+
config = config_factory()
171+
172+
with torch.no_grad():
173+
pipeline = VgfPipeline[input_t](
174+
test_helper.create_model(config),
175+
test_helper.create_dummy_inputs(config),
176+
aten_op=[],
177+
exir_op=[],
178+
use_to_edge_transform_and_lower=True,
179+
atol=5e-3,
180+
transform_passes=get_int64_to_int32_passes(),
181+
quantize=False,
182+
)
183+
pipeline.change_args(
184+
"check_count.exir",
185+
TestCLIPTextModelWithProjection.ops_after_partitioner_vgf_no_quantize,
186+
)
187+
pipeline.run()
188+
189+
190+
@common.SkipIfNoModelConverter
191+
@pytest.mark.parametrize(
192+
("config_factory", "atol"),
193+
(
194+
(get_tiny_sd35_large_text_encoder_config, 5.5e-2),
195+
(get_tiny_sd35_large_text_encoder_2_config, 6e-2),
196+
),
197+
ids=("text_encoder", "text_encoder_2"),
198+
)
199+
def test_clip_text_model_with_projection_vgf_quant(config_factory, atol):
200+
"""Run the CLIPTextModelWithProjection VGF quant test."""
201+
test_helper = TestCLIPTextModelWithProjection()
202+
config = config_factory()
203+
204+
with torch.no_grad():
205+
pipeline = VgfPipeline[input_t](
206+
test_helper.create_model(config),
207+
test_helper.create_dummy_inputs(config),
208+
aten_op=[],
209+
exir_op=[],
210+
use_to_edge_transform_and_lower=True,
211+
atol=atol,
212+
quantize=True,
213+
)
214+
pipeline.change_args(
215+
"check_count.exir",
216+
TestCLIPTextModelWithProjection.ops_after_partitioner_vgf_quantize,
217+
)
218+
pipeline.run()
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
# Copyright 2026 Arm Limited and/or its affiliates.
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
from typing import Tuple
7+
8+
import pytest
9+
import torch
10+
from executorch.backends.arm.test import common
11+
from executorch.backends.arm.test.models.stable_diffusion_3_5_large.test_configs_sd35_large import (
12+
get_tiny_sd35_large_transformer_config,
13+
)
14+
from executorch.backends.arm.test.tester.test_pipeline import (
15+
TosaPipelineFP,
16+
TosaPipelineINT,
17+
VgfPipeline,
18+
)
19+
from executorch.examples.models.stable_diffusion_3_5_large.model import (
20+
SD3TransformerWrapper,
21+
)
22+
23+
input_t4 = Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
24+
25+
26+
class TestSD3Transformer2DModel:
27+
"""Test helper for SD3.5 Large SD3Transformer2DModel config."""
28+
29+
ops_after_partitioner_FP = {
30+
"executorch_exir_dialects_edge__ops_aten_unsqueeze_copy_default": 1,
31+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default": 1,
32+
"torch.ops.higher_order.executorch_call_delegate": 1,
33+
}
34+
35+
ops_after_partitioner_INT = {
36+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default": 1,
37+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 1,
38+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 3,
39+
"torch.ops.higher_order.executorch_call_delegate": 1,
40+
}
41+
42+
ops_after_partitioner_vgf_quantize = {
43+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default": 1,
44+
"torch.ops.higher_order.executorch_call_delegate": 1,
45+
}
46+
ops_after_partitioner_vgf_no_quantize = ops_after_partitioner_FP
47+
48+
def create_config(self):
49+
"""Create a tiny SD3.5 Large-like MMDiT config for tests."""
50+
return get_tiny_sd35_large_transformer_config()
51+
52+
def create_dummy_inputs(
53+
self,
54+
batch_size: int = 2,
55+
latent_channels: int = 4,
56+
latent_size: int = 32,
57+
seq_length: int = 77,
58+
joint_attention_dim: int = 16,
59+
pooled_projection_dim: int = 32,
60+
max_timestep: int = 1000,
61+
dtype: torch.dtype = torch.float32,
62+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
63+
"""Create dummy inputs for the SD3Transformer2DModel tests."""
64+
# SD3.5 Large uses latent channels=16, latent size=128, T5 seq length=256,
65+
# joint_attention_dim=4096, and pooled_projection_dim=2048. Keep this
66+
# unit-test default smaller for TOSA runtime and VGF memory limits.
67+
return (
68+
torch.randn(
69+
batch_size,
70+
latent_channels,
71+
latent_size,
72+
latent_size,
73+
dtype=dtype,
74+
),
75+
torch.randint(low=0, high=max_timestep, size=(batch_size,)),
76+
torch.randn(
77+
batch_size,
78+
seq_length,
79+
joint_attention_dim,
80+
dtype=dtype,
81+
),
82+
torch.randn(batch_size, pooled_projection_dim, dtype=dtype),
83+
)
84+
85+
def create_model(self) -> SD3TransformerWrapper:
86+
"""Instantiate wrapped SD3Transformer2DModel for tests."""
87+
SD3Transformer2DModel = pytest.importorskip(
88+
"diffusers.models.transformers"
89+
).SD3Transformer2DModel
90+
return SD3TransformerWrapper(
91+
SD3Transformer2DModel(**self.create_config())
92+
).eval()
93+
94+
95+
def test_sd3_transformer_tosa_FP():
96+
"""Run the SD3Transformer2DModel TOSA FP test."""
97+
test_helper = TestSD3Transformer2DModel()
98+
99+
with torch.no_grad():
100+
pipeline = TosaPipelineFP[input_t4](
101+
test_helper.create_model(),
102+
test_helper.create_dummy_inputs(),
103+
aten_op=[],
104+
exir_op=[],
105+
use_to_edge_transform_and_lower=True,
106+
)
107+
pipeline.change_args(
108+
"check_count.exir", TestSD3Transformer2DModel.ops_after_partitioner_FP
109+
)
110+
pipeline.run()
111+
112+
113+
def test_sd3_transformer_tosa_INT():
114+
"""Run the SD3Transformer2DModel TOSA INT test."""
115+
test_helper = TestSD3Transformer2DModel()
116+
117+
with torch.no_grad():
118+
pipeline = TosaPipelineINT[input_t4](
119+
test_helper.create_model(),
120+
test_helper.create_dummy_inputs(),
121+
aten_op=[],
122+
exir_op=[],
123+
use_to_edge_transform_and_lower=True,
124+
frobenius_threshold=None,
125+
cosine_threshold=None,
126+
)
127+
pipeline.change_args(
128+
"check_count.exir", TestSD3Transformer2DModel.ops_after_partitioner_INT
129+
)
130+
pipeline.run()
131+
132+
133+
@common.SkipIfNoModelConverter
134+
def test_sd3_transformer_vgf_no_quant():
135+
"""Run the SD3Transformer2DModel VGF no-quant test."""
136+
test_helper = TestSD3Transformer2DModel()
137+
138+
with torch.no_grad():
139+
pipeline = VgfPipeline[input_t4](
140+
test_helper.create_model(),
141+
test_helper.create_dummy_inputs(),
142+
aten_op=[],
143+
exir_op=[],
144+
use_to_edge_transform_and_lower=True,
145+
quantize=False,
146+
)
147+
pipeline.change_args(
148+
"check_count.exir",
149+
TestSD3Transformer2DModel.ops_after_partitioner_vgf_no_quantize,
150+
)
151+
pipeline.run()
152+
153+
154+
@common.SkipIfNoModelConverter
155+
def test_sd3_transformer_vgf_quant():
156+
"""Run the SD3Transformer2DModel VGF quant test."""
157+
test_helper = TestSD3Transformer2DModel()
158+
159+
with torch.no_grad():
160+
pipeline = VgfPipeline[input_t4](
161+
test_helper.create_model(),
162+
test_helper.create_dummy_inputs(),
163+
aten_op=[],
164+
exir_op=[],
165+
use_to_edge_transform_and_lower=True,
166+
quantize=True,
167+
)
168+
pipeline.change_args(
169+
"check_count.exir",
170+
TestSD3Transformer2DModel.ops_after_partitioner_vgf_quantize,
171+
)
172+
pipeline.run()

0 commit comments

Comments
 (0)