Skip to content

Commit 0d8bb3d

Browse files
authored
Add profiling tests for MLPerfTiny models (pytorch#20622)
### Summary Add profiling tests for MLPerfTiny models. Fix a mapping issue for KWS model. ### Test plan Tests included. cc @robert-kalmar @JakeStevens @digantdesai @rascani
1 parent d03dee5 commit 0d8bb3d

2 files changed

Lines changed: 212 additions & 8 deletions

File tree

‎backends/nxp/backend/neutron_map.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -345,16 +345,18 @@ def get_tflite_to_neutron_map(self) -> dict[int, tuple[int, ...]]:
345345
tflite_to_neutron_dict = {}
346346
for tf_idx, tf_node in enumerate(self.tflite_nodes):
347347
subgraph_idxs = []
348+
location_shift = 0
348349
for subgraph in self.neutron_subgraphs:
349-
if (
350-
subgraph.num in self.neutron_graphs
351-
or subgraph.location in subgraph_idxs
352-
):
350+
if subgraph.num in self.neutron_graphs:
353351
continue
354352
for neutron_node in subgraph.nodes:
355353
if self._nodes_match_by_io(tf_node, neutron_node):
356-
subgraph_idxs.append(subgraph.location)
354+
for kernel in range(subgraph.kernels):
355+
subgraph_idxs.append(
356+
subgraph.location + location_shift + kernel
357+
)
357358
break
359+
location_shift += max(subgraph.kernels - 1, 0)
358360
# Filter subgraph_idxs to avoid mapping multiple parallel single-input nodes that consume the
359361
# same input tensor into the same TFLite node.
360362
subgraph_idxs = self._filter_single_input_nodes(tf_node.name, subgraph_idxs)

‎backends/nxp/tests/generic_tests/test_profiling.py‎

Lines changed: 205 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@
1717
from executorch.backends.nxp.tests.models import AvgPool2dModule, SoftmaxModule
1818
from executorch.backends.nxp.tests.nsys_testing import lower_run_compare
1919

20+
from executorch.examples.models.mlperf_tiny import (
21+
DeepAutoEncoder,
22+
DSCNNKWS,
23+
MobileNetV1025,
24+
ResNet8,
25+
)
26+
2027
from executorch.examples.nxp.experimental.cifar_net.cifar_net import CifarNetModel
2128

2229

@@ -39,7 +46,7 @@ def extract_map_from_logs(caplog):
3946
return None
4047

4148

42-
class ParallelPoolModel(torch.nn.Module):
49+
class SimpleParallelPoolModel(torch.nn.Module):
4350
def __init__(self, channels: int):
4451
super().__init__()
4552
self.conv_in = torch.nn.Conv2d(channels, channels, kernel_size=3, padding=1)
@@ -54,6 +61,28 @@ def forward(self, x):
5461
return x
5562

5663

64+
class ParallelPoolModel(torch.nn.Module):
65+
def __init__(self, ch=16):
66+
super().__init__()
67+
self.conv1 = torch.nn.Conv2d(ch, ch, 3, padding=1)
68+
self.bn1 = torch.nn.BatchNorm2d(ch)
69+
self.conv2 = torch.nn.Conv2d(ch, ch, 3, padding=1)
70+
self.maxpool = torch.nn.MaxPool2d(2)
71+
self.avgpool = torch.nn.AvgPool2d(2)
72+
self.conv_out = torch.nn.Conv2d(2 * ch, ch, 1)
73+
74+
def forward(self, x):
75+
residual = x
76+
x = self.conv1(x)
77+
x = self.bn1(x)
78+
x = torch.relu(x)
79+
x = self.conv2(x)
80+
x = x + residual # residual connection
81+
x = torch.cat((self.maxpool(x), self.avgpool(x)), dim=1) # parallel merge
82+
x = self.conv_out(x)
83+
return torch.relu(x)
84+
85+
5786
class TestProfiling:
5887
@pytest.mark.xfail(reason="SoftMax support PR is not merged so far.", strict=True)
5988
def test__softmax(self, caplog, request):
@@ -78,10 +107,10 @@ def test__softmax(self, caplog, request):
78107
3: (), # Neutron Dump
79108
}
80109

81-
def test__parallel_pool(self, caplog, request):
110+
def test__simple_parallel_pool(self, caplog, request):
82111
caplog.set_level(logging.INFO)
83112
input_shape = (1, 3, 32, 32)
84-
model = ParallelPoolModel(input_shape[1])
113+
model = SimpleParallelPoolModel(input_shape[1])
85114
lower_run_compare(
86115
model,
87116
input_shape,
@@ -156,3 +185,176 @@ def test__avg_pool(self, caplog, request):
156185
2: (2,), # Slice
157186
3: (), # Neutron Dump
158187
}
188+
189+
def test__parallel_pool(self, caplog, request):
190+
caplog.set_level(logging.INFO)
191+
input_shape = (1, 16, 32, 32)
192+
model = ParallelPoolModel(input_shape[1])
193+
lower_run_compare(
194+
model,
195+
input_shape,
196+
dlg_model_verifier=BaseGraphVerifier(1, []),
197+
request=request,
198+
output_comparator=NumericalStatsOutputComparator(),
199+
use_neutron_for_format_conversion=False,
200+
use_profiling=True,
201+
)
202+
neutron_map = extract_map_from_logs(caplog)
203+
assert neutron_map == {
204+
0: (8, 9), # Conv2DStandardV1 (Pad + Conv2d)
205+
1: (10,), # Conv2DStandardV1
206+
2: (11,), # Add
207+
3: (), # Conv2DDepthwiseV2 (AvgPool)
208+
4: (12,), # MaxPool
209+
5: (14,), # StridedSliceConcat
210+
6: (15, 16), # Conv2DPointwise (Conv2D + Relu)
211+
7: (), # Neutron Dump
212+
}
213+
214+
def test__resnet8(self, caplog, request):
215+
# Three-stage residual network for the MLPerf Tiny image-classification.
216+
caplog.set_level(logging.INFO)
217+
model = ResNet8()
218+
input_shape = (1, 3, 32, 32)
219+
220+
lower_run_compare(
221+
model,
222+
input_shape,
223+
dlg_model_verifier=BaseGraphVerifier(1, []),
224+
request=request,
225+
output_comparator=NumericalStatsOutputComparator(),
226+
use_neutron_for_format_conversion=False,
227+
use_profiling=True,
228+
)
229+
neutron_map = extract_map_from_logs(caplog)
230+
assert neutron_map == {
231+
0: (14, 15), # Conv2DStandardV2 (Pad + Conv)
232+
1: (17, 18), # Conv2DStandardV1 (Pad + Conv)
233+
2: (20,), # Conv2DStandardV1
234+
3: (21,), # Add
235+
4: (22,), # GlobalBiasScale (Relu)
236+
5: (28,), # Conv2DStandardV1
237+
6: (24, 25), # Conv2DStandardV1 (Pad + Conv)
238+
7: (27,), # Conv2DStandardV1
239+
8: (29,), # Add
240+
9: (30,), # GlobalBiasScale (Relu)
241+
10: (36,), # Conv2DStandardV1
242+
11: (32, 33), # Conv2DStandardV1 (Pad + Conv)
243+
12: (35,), # Conv2DStandardV1
244+
13: (37,), # Add
245+
14: (38,), # GlobalBiasScale (Relu)
246+
15: (), # GlobalAvgPool (Mean)
247+
16: (41,), # FullyConnected
248+
17: (), # Neutron Dump
249+
}
250+
251+
def test__ds_cnn(self, caplog, request):
252+
# Depthwise Separable CNN used for keyword spotting in MLCommons Tiny.
253+
caplog.set_level(logging.INFO)
254+
model = DSCNNKWS()
255+
input_shape = (1, 1, 49, 10)
256+
257+
lower_run_compare(
258+
model,
259+
input_shape,
260+
dlg_model_verifier=BaseGraphVerifier(1, []),
261+
request=request,
262+
output_comparator=NumericalStatsOutputComparator(),
263+
use_neutron_for_format_conversion=False,
264+
use_profiling=True,
265+
)
266+
neutron_map = extract_map_from_logs(caplog)
267+
assert neutron_map == {
268+
0: (14, 15), # Pad (Conv + Relu)
269+
1: (14, 15), # Conv2DStandardV2 (Conv + Relu)
270+
2: (18, 19), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
271+
3: (21, 22), # Conv2DPointwise (Conv + Relu)
272+
4: (24, 25), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
273+
5: (27, 28), # Conv2DDepthwiseV1 (Conv + Relu)
274+
6: (30, 31), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
275+
7: (33, 34), # Conv2DPointwise (Conv + Relu)
276+
8: (36, 37), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
277+
9: (39, 40), # Conv2DPointwise (Conv + Relu)
278+
10: (42,), # Conv2DDepthwiseDense (AvgPool)
279+
11: (44,), # FullyConnected
280+
12: (), # Neutron Dump
281+
}
282+
283+
def test__mobilenet_v1_025(self, caplog, request):
284+
# MobileNetV1 with width multiplier 0.25 for the Visual Wake Words.
285+
caplog.set_level(logging.INFO)
286+
model = MobileNetV1025()
287+
input_shape = (1, 3, 96, 96)
288+
289+
lower_run_compare(
290+
model,
291+
input_shape,
292+
dlg_model_verifier=BaseGraphVerifier(1, []),
293+
request=request,
294+
output_comparator=NumericalStatsOutputComparator(),
295+
use_neutron_for_format_conversion=False,
296+
use_profiling=True,
297+
)
298+
neutron_map = extract_map_from_logs(caplog)
299+
assert neutron_map == {
300+
0: (32, 33), # Conv2DStandardV2 (Conv + Relu)
301+
1: (35, 36), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
302+
2: (38, 39), # Conv2DPointwise (Conv + Relu)
303+
3: (41, 42), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
304+
4: (44, 45), # Conv2DPointwise (Conv + Relu)
305+
5: (47, 48), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
306+
6: (50, 51), # Conv2DPointwise (Conv + Relu)
307+
7: (53, 54), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
308+
8: (56, 57), # Conv2DPointwise (Conv + Relu)
309+
9: (59, 60), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
310+
10: (62, 63), # Conv2DPointwise (Conv + Relu)
311+
11: (65, 66), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
312+
12: (68, 69), # Conv2DPointwise (Conv + Relu)
313+
13: (71, 72), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
314+
14: (74, 75), # Conv2DPointwise (Conv + Relu)
315+
15: (77, 78), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
316+
16: (80, 81), # Conv2DPointwise (Conv + Relu)
317+
17: (83, 84), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
318+
18: (86, 87), # Conv2DPointwise (Conv + Relu)
319+
19: (89, 90), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
320+
20: (92, 93), # Conv2DPointwise (Conv + Relu)
321+
21: (95, 96), # Conv2DDepthwiseV1 (DepthwiseConv + Relu)
322+
22: (98, 99), # Conv2DPointwise (Conv + Relu)
323+
23: (101, 102), # Conv2DDepthwiseDense (DepthwiseConv + Relu)
324+
24: (104, 105), # Conv2DPointwise (Conv + Relu)
325+
25: (107, 108), # Conv2DDepthwiseDense (DepthwiseConv + Relu)
326+
26: (110, 111), # Conv2DPointwise (Conv + Relu)
327+
27: (), # Mean (GlobalAvgPool)
328+
28: (114,), # FullyConnected
329+
29: (), # Neutron Dump
330+
}
331+
332+
def test__deep_autoencoder(self, caplog, request):
333+
# MLPerf Tiny anomaly detection deep autoencoder.
334+
caplog.set_level(logging.INFO)
335+
model = DeepAutoEncoder()
336+
input_shape = (1, 640)
337+
338+
lower_run_compare(
339+
model,
340+
input_shape,
341+
dlg_model_verifier=BaseGraphVerifier(1, []),
342+
request=request,
343+
output_comparator=NumericalStatsOutputComparator(),
344+
use_neutron_for_format_conversion=False,
345+
use_profiling=True,
346+
)
347+
neutron_map = extract_map_from_logs(caplog)
348+
assert neutron_map == {
349+
0: (22, 23), # FullyConnected (FullyConnected + Relu)
350+
1: (24, 25), # FullyConnected (FullyConnected + Relu)
351+
2: (26, 27), # FullyConnected (FullyConnected + Relu)
352+
3: (28, 29), # FullyConnected (FullyConnected + Relu)
353+
4: (30, 31), # FullyConnected (FullyConnected + Relu)
354+
5: (32, 33), # FullyConnected (FullyConnected + Relu)
355+
6: (34, 35), # FullyConnected (FullyConnected + Relu)
356+
7: (36, 37), # FullyConnected (FullyConnected + Relu)
357+
8: (38, 39), # FullyConnected (FullyConnected + Relu)
358+
9: (40,), # FullyConnected
359+
10: (), # Neutron Dump
360+
}

0 commit comments

Comments
 (0)