|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the BSD-style license found in the |
| 5 | +# LICENSE file in the root directory of this source tree. |
| 6 | + |
| 7 | +"""Cortex-M flow for the backend test suite. |
| 8 | +
|
| 9 | +Every case is serialized and executed on the Corstone-300 FVP with the |
| 10 | +semihosting runner from backends/cortex_m/test/build_test_runner.sh. That runner |
| 11 | +registers the cortex_m kernels along with the portable fallbacks listed in its |
| 12 | +ops_list, so an operator the backend does not lower still runs when the list |
| 13 | +happens to carry one. |
| 14 | +
|
| 15 | +The kernels are CMSIS-NN compiled for the device and cannot run on a host CPU, |
| 16 | +so the simulator is the only place to execute them. The Arm Ethos-U flows reach |
| 17 | +the Corstone FVP the same way. flow.py leaves the flow unregistered when the FVP |
| 18 | +is not on PATH. |
| 19 | +
|
| 20 | +Comparison uses the suite's fixed atol of 1e-1, which is not tuned for int8, so |
| 21 | +a mismatch is not by itself evidence of a lowering bug. |
| 22 | +""" |
| 23 | + |
| 24 | +import torch |
| 25 | + |
| 26 | +from executorch.backends.cortex_m.test.tester import CortexMQuantize, CortexMTester |
| 27 | +from executorch.backends.test.suite.flow import TestFlow |
| 28 | + |
| 29 | + |
| 30 | +def _create_cortex_m_tester(model, inputs, **kwargs) -> CortexMTester: |
| 31 | + # The CMSIS-NN kernels are NHWC, and CortexMConv2DCheck rejects any pattern |
| 32 | + # whose tensors are not channels_last. The suite hands over contiguous |
| 33 | + # tensors, which silently costs every convolution in the model. |
| 34 | + inputs = tuple( |
| 35 | + ( |
| 36 | + t.to(memory_format=torch.channels_last) |
| 37 | + if isinstance(t, torch.Tensor) and t.dim() == 4 |
| 38 | + else t |
| 39 | + ) |
| 40 | + for t in inputs |
| 41 | + ) |
| 42 | + # The Serialize stage is also the stage that invokes the ELF, so its timeout |
| 43 | + # is the FVP's --timelimit rather than a serialization budget, and 120s is |
| 44 | + # not enough for an ImageNet-sized model on an M55 with no NPU. Kept under |
| 45 | + # the suite conftest's 1200s pytest timeout so the FVP reports the overrun |
| 46 | + # instead of pytest killing the process. |
| 47 | + return CortexMTester(model, inputs, timeout=900, **kwargs) |
| 48 | + |
| 49 | + |
| 50 | +# Models that cannot run on the FVP runner at all, so there is nothing for the |
| 51 | +# report to record. A skipped test produces no row, which is why the sizes are |
| 52 | +# written down here. |
| 53 | +CORTEX_M_SKIPS = [ |
| 54 | + # Over the runner's 60 MiB pool even fully quantized: 86.6M and 68.9M |
| 55 | + # parameters. Raising ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE |
| 56 | + # is the only thing that would change that. |
| 57 | + "test_vit_b_16", |
| 58 | + "test_wide_resnet50_2", |
| 59 | + # The program is under a megabyte; it is the activation arena that does not |
| 60 | + # fit, and it crosses the pool at a sequence length of about 353. The test |
| 61 | + # draws lengths from randint(1, 400), so it fits roughly one run in three. |
| 62 | + "test_conformer", |
| 63 | +] |
| 64 | + |
| 65 | +# Models that run to completion and fail. They stay in the suite so the report |
| 66 | +# keeps recording how they fail; the marker only keeps a known state from |
| 67 | +# turning the job red, and it is strict, so anything fixed reports XPASS and |
| 68 | +# this list has to shrink. |
| 69 | +CORTEX_M_XFAILS = [ |
| 70 | + # Whatever a model leaves unlowered runs on the portable kernels compiled |
| 71 | + # into the runner, and ops_list in build_test_runner.sh is a fixed list. |
| 72 | + "test_convnext_small", |
| 73 | + "test_densenet161", |
| 74 | + "test_maxvit_t", |
| 75 | + "test_mnasnet1_0", |
| 76 | + "test_shufflenet_v2_x1_0", |
| 77 | + # 4.00 bytes per parameter: none of the convolutions lower, so the weights |
| 78 | + # stay fp32 and the 100 MiB program overruns the pool. Lowering conv1d |
| 79 | + # brings it to 26 MiB, which fits. |
| 80 | + "test_wav2letter", |
| 81 | + # Runs on target and does not match. |
| 82 | + "test_inception_v3", |
| 83 | + "test_resnet50", |
| 84 | + "test_squeezenet1_1", |
| 85 | + # AtenToCortexMPass rejects the graph. |
| 86 | + "test_swin_v2_t", |
| 87 | +] |
| 88 | + |
| 89 | + |
| 90 | +def _create_cortex_m_flow() -> TestFlow: |
| 91 | + return TestFlow( |
| 92 | + "cortex_m", |
| 93 | + backend="cortex_m", |
| 94 | + tester_factory=_create_cortex_m_tester, |
| 95 | + quantize=True, |
| 96 | + quantize_stage_factory=CortexMQuantize, |
| 97 | + is_delegated=False, |
| 98 | + param_skip_reasons={ |
| 99 | + "use_dynamic_shapes": { |
| 100 | + True: "Cortex-M lowers for a fixed shape; the CMSIS-NN kernels " |
| 101 | + "take their dimensions from the graph." |
| 102 | + } |
| 103 | + }, |
| 104 | + skip_patterns=CORTEX_M_SKIPS, |
| 105 | + xfail_patterns=CORTEX_M_XFAILS, |
| 106 | + ) |
| 107 | + |
| 108 | + |
| 109 | +CORTEX_M_TEST_FLOW = _create_cortex_m_flow() |
0 commit comments