diff --git a/backends/qualcomm/export_utils.py b/backends/qualcomm/export_utils.py index 9059e53324d..7241c0452dc 100644 --- a/backends/qualcomm/export_utils.py +++ b/backends/qualcomm/export_utils.py @@ -164,7 +164,11 @@ def __post_init__(self): np.random.seed(self.seed) random.seed(self.seed) - self.backend = get_backend_type(self.backend) + self.backend = ( + get_backend_type(self.backend) + if isinstance(self.backend, str) + else self.backend + ) self.skip_delegate_node_ids, self.skip_delegate_node_ops = ( self._parse_skip_delegation_node( self.skip_delegate_node_ids, self.skip_delegate_node_ops diff --git a/backends/qualcomm/tests/rework/conftest.py b/backends/qualcomm/tests/rework/conftest.py index 9730b120c16..667da97e996 100644 --- a/backends/qualcomm/tests/rework/conftest.py +++ b/backends/qualcomm/tests/rework/conftest.py @@ -9,6 +9,7 @@ import logging import os import random +import re import subprocess import tempfile import time @@ -19,6 +20,7 @@ from contextlib import contextmanager from dataclasses import dataclass from functools import partial +from pathlib import Path from typing import Any, List, Tuple import numpy as np @@ -38,6 +40,12 @@ SimpleADB, to_edge_transform_and_lower_to_qnn, ) +from executorch.backends.qualcomm.serialization.qc_schema import ( + _soc_info_table, + HtpArch, + LpaiHardwareVersion, + QcomChipset, +) from executorch.examples.qualcomm.utils import make_output_dir from executorch.exir.passes.memory_planning_pass import MemoryPlanningPass from executorch.exir.program import ExecutorchProgramManager @@ -560,6 +568,176 @@ def validate(): ) +def _get_htp_arch_from_path(path: Path) -> HtpArch | None: + try: + rework_index = path.parts.index("rework") + except ValueError: + return None + + if path.parts[rework_index + 1 : rework_index + 3] not in ( + ("htp", "e2e"), + ("htp", "feature"), + ): + return None + + for part in path.parts[rework_index + 3 : -1]: + match = re.fullmatch(r"v(\d+)", part) + if match is None: + continue + try: + return HtpArch(int(match.group(1))) + except ValueError as error: + raise pytest.UsageError( + f"Unknown HTP architecture directory {part} in {path}." + ) from error + return None + + +def _get_lpai_hardware_version_from_path( + path: Path, +) -> LpaiHardwareVersion | None: + try: + rework_index = path.parts.index("rework") + except ValueError: + return None + + if path.parts[rework_index + 1 : rework_index + 3] not in ( + ("lpai", "e2e"), + ("lpai", "feature"), + ): + return None + + for part in path.parts[rework_index + 3 : -1]: + match = re.fullmatch(r"v(\d+)", part) + if match is None: + continue + try: + hardware_version = LpaiHardwareVersion(int(match.group(1))) + except ValueError as error: + raise pytest.UsageError( + f"Unknown LPAI hardware version directory {part} in {path}." + ) from error + if hardware_version == LpaiHardwareVersion.NONE: + raise pytest.UsageError( + f"Unknown LPAI hardware version directory {part} in {path}." + ) + return hardware_version + return None + + +def _validate_htp_test_path(path: Path, soc_model: str | None) -> None: + expected_arch = _get_htp_arch_from_path(path) + if expected_arch is None or soc_model is None: + return + + try: + soc_info = _soc_info_table[QcomChipset[soc_model]] + except KeyError as error: + raise pytest.UsageError(f"Unknown --soc_model {soc_model}.") from error + + actual_arch = soc_info.htp_info.htp_arch + if actual_arch == HtpArch.NONE: + raise pytest.UsageError(f"--soc_model {soc_model} does not support HTP.") + if actual_arch != expected_arch: + raise pytest.UsageError( + f"--soc_model {soc_model} resolves to HTP v{actual_arch.value}, but " + f"{path} targets HTP v{expected_arch.value}. Select the v{actual_arch.value} " + f"test path or use an HTP v{expected_arch.value} SoC." + ) + + +def _validate_lpai_test_path(path: Path, soc_model: str | None) -> None: + expected_version = _get_lpai_hardware_version_from_path(path) + if expected_version is None or soc_model is None: + return + + try: + soc_info = _soc_info_table[QcomChipset[soc_model]] + except KeyError as error: + raise pytest.UsageError(f"Unknown --soc_model {soc_model}.") from error + + if ( + soc_info.lpai_info is None + or soc_info.lpai_info.lpai_hardware_version == LpaiHardwareVersion.NONE + ): + raise pytest.UsageError(f"--soc_model {soc_model} does not support LPAI.") + + actual_version = soc_info.lpai_info.lpai_hardware_version + if actual_version != expected_version: + raise pytest.UsageError( + f"--soc_model {soc_model} resolves to LPAI v{actual_version.value}, but " + f"{path} targets LPAI v{expected_version.value}. Select the " + f"v{actual_version.value} test path or use an LPAI " + f"v{expected_version.value} SoC." + ) + + +def pytest_collect_file(file_path, parent): + path = Path(file_path) + soc_model = parent.config.getoption("soc_model") + _validate_htp_test_path(path, soc_model) + _validate_lpai_test_path(path, soc_model) + + +def get_backend_str(qnn_config) -> str: + return { + QnnExecuTorchBackendType.kHtpBackend: "htp", + QnnExecuTorchBackendType.kGpuBackend: "gpu", + QnnExecuTorchBackendType.kLpaiBackend: "lpai", + }.get(qnn_config.backend) + + +def get_ipc(qnn_config): + ip = qnn_config.ip or "localhost" + port = qnn_config.port if (qnn_config.port and qnn_config.port > 0) else 8888 + return ip, port + + +def add_default_cmds(cmds: List, qnn_config) -> None: + ip, port = get_ipc(qnn_config) + cmds.extend( + [ + "--soc_model", + qnn_config.soc_model, + "--target", + qnn_config.target, + "--ip", + ip, + "--port", + str(port), + "--seed", + "1126", + "--backend", + get_backend_str(qnn_config), + ] + ) + if qnn_config.compile_only: + cmds.extend(["--compile_only"]) + elif qnn_config.device: + cmds.extend(["--device", qnn_config.device]) + + if qnn_config.host: + cmds.extend(["--host", qnn_config.host]) + elif qnn_config.enable_x86_64: + cmds.extend(["--enable_x86_64"]) + + if getattr(qnn_config, "online_prepare", False): + cmds.extend(["--online_prepare"]) + + if getattr(qnn_config, "shared_buffer", False): + cmds.extend(["--shared_buffer"]) + + if qnn_config.pre_gen_pte: + cmds.extend(["--pre_gen_pte", qnn_config.pre_gen_pte]) + + +def require_paths(paths: dict) -> None: + """Fail loudly when a provided path does not exist on disk.""" + for name, path in paths.items(): + if not path or not os.path.exists(path): + pytest.fail(f"Required artifact not found: {name}") + + def pytest_addoption(parser): setup_common_args_and_variables(parser=parser) parser.addoption( @@ -567,6 +745,73 @@ def pytest_addoption(parser): type=str, help="Specify report path while testing", ) + parser.addoption( + "--executorch_root", + type=str, + default=None, + help="Root directory of the ExecuTorch repository.", + ) + parser.addoption( + "--artifact_dir", + type=str, + default=None, + help="Directory for storing compiled artifacts.", + ) + parser.addoption( + "--image_dataset", + type=str, + default=None, + help="Path to the ImageNet-style image dataset.", + ) + parser.addoption( + "--sentence_dataset", + type=str, + default=None, + help="Path to the sentence / NLP dataset.", + ) + parser.addoption( + "--qa_dataset", + type=str, + default=None, + help="Path to the QA dataset (used by e.g. T5).", + ) + parser.addoption( + "--pretrained_weight", + type=str, + default=None, + help="Path to pretrained model weights.", + ) + parser.addoption( + "--oss_repo", + type=str, + default=None, + help="Path to the OSS repository required by some scripts.", + ) + parser.addoption( + "--model_name", + type=str, + default=None, + help="Model name (e.g. llama3_2-1b_instruct) for LLM / multimodality tests.", + ) + parser.addoption( + "--llama_artifacts", + type=str, + default=None, + help="Directory containing llama checkpoint/params/tokenizer files.", + ) + parser.addoption( + "--static_llm_eval_method", + type=str, + default=None, + choices=["wikitext_ppl", "hellaswag_acc_norm", "sqnr"], + help="Evaluation method for the static LLM test.", + ) + parser.addoption( + "--use_fp16", + action="store_true", + default=False, + help="Use FP16 weights (used by test_llama_stories_110m).", + ) def pytest_terminal_summary(terminalreporter, exitstatus, config): diff --git a/backends/qualcomm/tests/rework/htp/e2e/genai/v68/test.py b/backends/qualcomm/tests/rework/htp/e2e/genai/v68/test.py index b5f86874fd4..99e17e5e8fb 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/genai/v68/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/genai/v68/test.py @@ -3,3 +3,46 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + + +from executorch.backends.qualcomm.tests.rework.src.genai import * # noqa: F403 + + +def test_static_llm_model(request): + StaticLLM.test(request) # noqa: F405 + + +def test_static_llm_eval_limit(request): + StaticLLMEvalLimit.test(request) # noqa: F405 + + +def test_codegen2_1b(request): + Codegen2.test(request) # noqa: F405 + + +def test_llama_stories_260k(request): + LlamaStories260K.test(request) # noqa: F405 + + +def test_llama_stories_110m(request): + LlamaStories110M.test(request) # noqa: F405 + + +def test_attention_sink(request): + AttentionSink.test(request) # noqa: F405 + + +def test_hf_causal_lm(request): + HFCausalLM.test(request) # noqa: F405 + + +def test_static_llm_qat(request): + StaticLLMQAT.test(request) # noqa: F405 + + +def test_static_asr(request): + StaticASR.test(request) # noqa: F405 + + +def test_static_vlm(request): + StaticVLM.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/genai/v69/test.py b/backends/qualcomm/tests/rework/htp/e2e/genai/v69/test.py index b5f86874fd4..99e17e5e8fb 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/genai/v69/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/genai/v69/test.py @@ -3,3 +3,46 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + + +from executorch.backends.qualcomm.tests.rework.src.genai import * # noqa: F403 + + +def test_static_llm_model(request): + StaticLLM.test(request) # noqa: F405 + + +def test_static_llm_eval_limit(request): + StaticLLMEvalLimit.test(request) # noqa: F405 + + +def test_codegen2_1b(request): + Codegen2.test(request) # noqa: F405 + + +def test_llama_stories_260k(request): + LlamaStories260K.test(request) # noqa: F405 + + +def test_llama_stories_110m(request): + LlamaStories110M.test(request) # noqa: F405 + + +def test_attention_sink(request): + AttentionSink.test(request) # noqa: F405 + + +def test_hf_causal_lm(request): + HFCausalLM.test(request) # noqa: F405 + + +def test_static_llm_qat(request): + StaticLLMQAT.test(request) # noqa: F405 + + +def test_static_asr(request): + StaticASR.test(request) # noqa: F405 + + +def test_static_vlm(request): + StaticVLM.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/genai/v73/test.py b/backends/qualcomm/tests/rework/htp/e2e/genai/v73/test.py index b5f86874fd4..99e17e5e8fb 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/genai/v73/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/genai/v73/test.py @@ -3,3 +3,46 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + + +from executorch.backends.qualcomm.tests.rework.src.genai import * # noqa: F403 + + +def test_static_llm_model(request): + StaticLLM.test(request) # noqa: F405 + + +def test_static_llm_eval_limit(request): + StaticLLMEvalLimit.test(request) # noqa: F405 + + +def test_codegen2_1b(request): + Codegen2.test(request) # noqa: F405 + + +def test_llama_stories_260k(request): + LlamaStories260K.test(request) # noqa: F405 + + +def test_llama_stories_110m(request): + LlamaStories110M.test(request) # noqa: F405 + + +def test_attention_sink(request): + AttentionSink.test(request) # noqa: F405 + + +def test_hf_causal_lm(request): + HFCausalLM.test(request) # noqa: F405 + + +def test_static_llm_qat(request): + StaticLLMQAT.test(request) # noqa: F405 + + +def test_static_asr(request): + StaticASR.test(request) # noqa: F405 + + +def test_static_vlm(request): + StaticVLM.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/genai/v75/test.py b/backends/qualcomm/tests/rework/htp/e2e/genai/v75/test.py index b5f86874fd4..99e17e5e8fb 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/genai/v75/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/genai/v75/test.py @@ -3,3 +3,46 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + + +from executorch.backends.qualcomm.tests.rework.src.genai import * # noqa: F403 + + +def test_static_llm_model(request): + StaticLLM.test(request) # noqa: F405 + + +def test_static_llm_eval_limit(request): + StaticLLMEvalLimit.test(request) # noqa: F405 + + +def test_codegen2_1b(request): + Codegen2.test(request) # noqa: F405 + + +def test_llama_stories_260k(request): + LlamaStories260K.test(request) # noqa: F405 + + +def test_llama_stories_110m(request): + LlamaStories110M.test(request) # noqa: F405 + + +def test_attention_sink(request): + AttentionSink.test(request) # noqa: F405 + + +def test_hf_causal_lm(request): + HFCausalLM.test(request) # noqa: F405 + + +def test_static_llm_qat(request): + StaticLLMQAT.test(request) # noqa: F405 + + +def test_static_asr(request): + StaticASR.test(request) # noqa: F405 + + +def test_static_vlm(request): + StaticVLM.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/genai/v79/test.py b/backends/qualcomm/tests/rework/htp/e2e/genai/v79/test.py index b5f86874fd4..99e17e5e8fb 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/genai/v79/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/genai/v79/test.py @@ -3,3 +3,46 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + + +from executorch.backends.qualcomm.tests.rework.src.genai import * # noqa: F403 + + +def test_static_llm_model(request): + StaticLLM.test(request) # noqa: F405 + + +def test_static_llm_eval_limit(request): + StaticLLMEvalLimit.test(request) # noqa: F405 + + +def test_codegen2_1b(request): + Codegen2.test(request) # noqa: F405 + + +def test_llama_stories_260k(request): + LlamaStories260K.test(request) # noqa: F405 + + +def test_llama_stories_110m(request): + LlamaStories110M.test(request) # noqa: F405 + + +def test_attention_sink(request): + AttentionSink.test(request) # noqa: F405 + + +def test_hf_causal_lm(request): + HFCausalLM.test(request) # noqa: F405 + + +def test_static_llm_qat(request): + StaticLLMQAT.test(request) # noqa: F405 + + +def test_static_asr(request): + StaticASR.test(request) # noqa: F405 + + +def test_static_vlm(request): + StaticVLM.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/genai/v81/test.py b/backends/qualcomm/tests/rework/htp/e2e/genai/v81/test.py index b5f86874fd4..99e17e5e8fb 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/genai/v81/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/genai/v81/test.py @@ -3,3 +3,46 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + + +from executorch.backends.qualcomm.tests.rework.src.genai import * # noqa: F403 + + +def test_static_llm_model(request): + StaticLLM.test(request) # noqa: F405 + + +def test_static_llm_eval_limit(request): + StaticLLMEvalLimit.test(request) # noqa: F405 + + +def test_codegen2_1b(request): + Codegen2.test(request) # noqa: F405 + + +def test_llama_stories_260k(request): + LlamaStories260K.test(request) # noqa: F405 + + +def test_llama_stories_110m(request): + LlamaStories110M.test(request) # noqa: F405 + + +def test_attention_sink(request): + AttentionSink.test(request) # noqa: F405 + + +def test_hf_causal_lm(request): + HFCausalLM.test(request) # noqa: F405 + + +def test_static_llm_qat(request): + StaticLLMQAT.test(request) # noqa: F405 + + +def test_static_asr(request): + StaticASR.test(request) # noqa: F405 + + +def test_static_vlm(request): + StaticVLM.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/oss/v68/test.py b/backends/qualcomm/tests/rework/htp/e2e/oss/v68/test.py index b5f86874fd4..5f3b3438f36 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/oss/v68/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/oss/v68/test.py @@ -3,3 +3,189 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + +import pytest + +from executorch.backends.qualcomm.tests.rework.src.oss import * # noqa: F403 + + +# -- TestExampleScript (examples/qualcomm/scripts/) -- + + +def test_mobilenet_v2(request): + MobileNetV2.test(request) # noqa: F405 + + +def test_mobilenet_v3(request): + MobileNetV3.test(request) # noqa: F405 + + +def test_inception_v3(request): + InceptionV3.test(request) # noqa: F405 + + +def test_inception_v4(request): + InceptionV4.test(request) # noqa: F405 + + +def test_vit(request): + Vit.test(request) # noqa: F405 + + +def test_edsr(request): + Edsr.test(request) # noqa: F405 + + +def test_deeplab_v3(request): + DeepLabV3.test(request) # noqa: F405 + + +@pytest.mark.skip("dynamic shape inputs appear in recent torch.export.export") +def test_mobilebert(request): + MobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("eagar mode fake quant works well, need further investigation") +def test_ptq_mobilebert(request): + PtqMobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("encountered undefined symbol in mainline, reopen once resolved") +def test_wav2letter(request): + Wav2Letter.test(request) # noqa: F405 + + +def test_export_example(request): + ExportExample.test(request) # noqa: F405 + + +# -- TestExampleOssScript (examples/qualcomm/oss_scripts/) -- + + +def test_albert(request): + Albert.test(request) # noqa: F405 + + +def test_bert(request): + Bert.test(request) # noqa: F405 + + +def test_conv_former(request): + ConvFormer.test(request) # noqa: F405 + + +def test_convnext_small(request): + ConvNextSmall.test(request) # noqa: F405 + + +def test_cvt(request): + Cvt.test(request) # noqa: F405 + + +def test_deit(request): + Deit.test(request) # noqa: F405 + + +def test_depthanything_v2_small(request): + DepthAnythingV2Small.test(request) # noqa: F405 + + +def test_dino_v2(request): + DinoV2.test(request) # noqa: F405 + + +def test_distilbert(request): + Distilbert.test(request) # noqa: F405 + + +def test_dit(request): + Dit.test(request) # noqa: F405 + + +def test_efficientnet(request): + EfficientNet.test(request) # noqa: F405 + + +def test_efficientSAM(request): + EfficientSAM.test(request) # noqa: F405 + + +def test_esrgan(request): + Esrgan.test(request) # noqa: F405 + + +@pytest.mark.skip("Bad accuracy on source model since transformers v5") +def test_eurobert(request): + Eurobert.test(request) # noqa: F405 + + +def test_fastvit(request): + FastVit.test(request) # noqa: F405 + + +def test_fbnet(request): + FbNet.test(request) # noqa: F405 + + +def test_focalnet(request): + FocalNet.test(request) # noqa: F405 + + +def test_gMLP(request): + GMLP.test(request) # noqa: F405 + + +def test_maxvit_t(request): + MaxVitT.test(request) # noqa: F405 + + +def test_mobilevit_v2(request): + MobileViTV2.test(request) # noqa: F405 + + +def test_mobilevit_v1(request): + MobileViTV1.test(request) # noqa: F405 + + +def test_pvt(request): + Pvt.test(request) # noqa: F405 + + +def test_regnet(request): + RegNet.test(request) # noqa: F405 + + +def test_retinanet(request): + RetinaNet.test(request) # noqa: F405 + + +def test_roberta(request): + Roberta.test(request) # noqa: F405 + + +def test_squeezenet(request): + SqueezeNet.test(request) # noqa: F405 + + +def test_ssd300_vgg16(request): + Ssd300Vgg16.test(request) # noqa: F405 + + +def test_swin_transformer(request): + SwinTransformer.test(request) # noqa: F405 + + +def test_swin_v2_t(request): + SwinV2T.test(request) # noqa: F405 + + +def test_t5(request): + T5.test(request) # noqa: F405 + + +def test_vit_b_16(request): + VitB16.test(request) # noqa: F405 + + +def test_whisper(request): + Whisper.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/oss/v69/test.py b/backends/qualcomm/tests/rework/htp/e2e/oss/v69/test.py index b5f86874fd4..5f3b3438f36 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/oss/v69/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/oss/v69/test.py @@ -3,3 +3,189 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + +import pytest + +from executorch.backends.qualcomm.tests.rework.src.oss import * # noqa: F403 + + +# -- TestExampleScript (examples/qualcomm/scripts/) -- + + +def test_mobilenet_v2(request): + MobileNetV2.test(request) # noqa: F405 + + +def test_mobilenet_v3(request): + MobileNetV3.test(request) # noqa: F405 + + +def test_inception_v3(request): + InceptionV3.test(request) # noqa: F405 + + +def test_inception_v4(request): + InceptionV4.test(request) # noqa: F405 + + +def test_vit(request): + Vit.test(request) # noqa: F405 + + +def test_edsr(request): + Edsr.test(request) # noqa: F405 + + +def test_deeplab_v3(request): + DeepLabV3.test(request) # noqa: F405 + + +@pytest.mark.skip("dynamic shape inputs appear in recent torch.export.export") +def test_mobilebert(request): + MobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("eagar mode fake quant works well, need further investigation") +def test_ptq_mobilebert(request): + PtqMobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("encountered undefined symbol in mainline, reopen once resolved") +def test_wav2letter(request): + Wav2Letter.test(request) # noqa: F405 + + +def test_export_example(request): + ExportExample.test(request) # noqa: F405 + + +# -- TestExampleOssScript (examples/qualcomm/oss_scripts/) -- + + +def test_albert(request): + Albert.test(request) # noqa: F405 + + +def test_bert(request): + Bert.test(request) # noqa: F405 + + +def test_conv_former(request): + ConvFormer.test(request) # noqa: F405 + + +def test_convnext_small(request): + ConvNextSmall.test(request) # noqa: F405 + + +def test_cvt(request): + Cvt.test(request) # noqa: F405 + + +def test_deit(request): + Deit.test(request) # noqa: F405 + + +def test_depthanything_v2_small(request): + DepthAnythingV2Small.test(request) # noqa: F405 + + +def test_dino_v2(request): + DinoV2.test(request) # noqa: F405 + + +def test_distilbert(request): + Distilbert.test(request) # noqa: F405 + + +def test_dit(request): + Dit.test(request) # noqa: F405 + + +def test_efficientnet(request): + EfficientNet.test(request) # noqa: F405 + + +def test_efficientSAM(request): + EfficientSAM.test(request) # noqa: F405 + + +def test_esrgan(request): + Esrgan.test(request) # noqa: F405 + + +@pytest.mark.skip("Bad accuracy on source model since transformers v5") +def test_eurobert(request): + Eurobert.test(request) # noqa: F405 + + +def test_fastvit(request): + FastVit.test(request) # noqa: F405 + + +def test_fbnet(request): + FbNet.test(request) # noqa: F405 + + +def test_focalnet(request): + FocalNet.test(request) # noqa: F405 + + +def test_gMLP(request): + GMLP.test(request) # noqa: F405 + + +def test_maxvit_t(request): + MaxVitT.test(request) # noqa: F405 + + +def test_mobilevit_v2(request): + MobileViTV2.test(request) # noqa: F405 + + +def test_mobilevit_v1(request): + MobileViTV1.test(request) # noqa: F405 + + +def test_pvt(request): + Pvt.test(request) # noqa: F405 + + +def test_regnet(request): + RegNet.test(request) # noqa: F405 + + +def test_retinanet(request): + RetinaNet.test(request) # noqa: F405 + + +def test_roberta(request): + Roberta.test(request) # noqa: F405 + + +def test_squeezenet(request): + SqueezeNet.test(request) # noqa: F405 + + +def test_ssd300_vgg16(request): + Ssd300Vgg16.test(request) # noqa: F405 + + +def test_swin_transformer(request): + SwinTransformer.test(request) # noqa: F405 + + +def test_swin_v2_t(request): + SwinV2T.test(request) # noqa: F405 + + +def test_t5(request): + T5.test(request) # noqa: F405 + + +def test_vit_b_16(request): + VitB16.test(request) # noqa: F405 + + +def test_whisper(request): + Whisper.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/oss/v73/test.py b/backends/qualcomm/tests/rework/htp/e2e/oss/v73/test.py index b5f86874fd4..5f3b3438f36 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/oss/v73/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/oss/v73/test.py @@ -3,3 +3,189 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + +import pytest + +from executorch.backends.qualcomm.tests.rework.src.oss import * # noqa: F403 + + +# -- TestExampleScript (examples/qualcomm/scripts/) -- + + +def test_mobilenet_v2(request): + MobileNetV2.test(request) # noqa: F405 + + +def test_mobilenet_v3(request): + MobileNetV3.test(request) # noqa: F405 + + +def test_inception_v3(request): + InceptionV3.test(request) # noqa: F405 + + +def test_inception_v4(request): + InceptionV4.test(request) # noqa: F405 + + +def test_vit(request): + Vit.test(request) # noqa: F405 + + +def test_edsr(request): + Edsr.test(request) # noqa: F405 + + +def test_deeplab_v3(request): + DeepLabV3.test(request) # noqa: F405 + + +@pytest.mark.skip("dynamic shape inputs appear in recent torch.export.export") +def test_mobilebert(request): + MobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("eagar mode fake quant works well, need further investigation") +def test_ptq_mobilebert(request): + PtqMobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("encountered undefined symbol in mainline, reopen once resolved") +def test_wav2letter(request): + Wav2Letter.test(request) # noqa: F405 + + +def test_export_example(request): + ExportExample.test(request) # noqa: F405 + + +# -- TestExampleOssScript (examples/qualcomm/oss_scripts/) -- + + +def test_albert(request): + Albert.test(request) # noqa: F405 + + +def test_bert(request): + Bert.test(request) # noqa: F405 + + +def test_conv_former(request): + ConvFormer.test(request) # noqa: F405 + + +def test_convnext_small(request): + ConvNextSmall.test(request) # noqa: F405 + + +def test_cvt(request): + Cvt.test(request) # noqa: F405 + + +def test_deit(request): + Deit.test(request) # noqa: F405 + + +def test_depthanything_v2_small(request): + DepthAnythingV2Small.test(request) # noqa: F405 + + +def test_dino_v2(request): + DinoV2.test(request) # noqa: F405 + + +def test_distilbert(request): + Distilbert.test(request) # noqa: F405 + + +def test_dit(request): + Dit.test(request) # noqa: F405 + + +def test_efficientnet(request): + EfficientNet.test(request) # noqa: F405 + + +def test_efficientSAM(request): + EfficientSAM.test(request) # noqa: F405 + + +def test_esrgan(request): + Esrgan.test(request) # noqa: F405 + + +@pytest.mark.skip("Bad accuracy on source model since transformers v5") +def test_eurobert(request): + Eurobert.test(request) # noqa: F405 + + +def test_fastvit(request): + FastVit.test(request) # noqa: F405 + + +def test_fbnet(request): + FbNet.test(request) # noqa: F405 + + +def test_focalnet(request): + FocalNet.test(request) # noqa: F405 + + +def test_gMLP(request): + GMLP.test(request) # noqa: F405 + + +def test_maxvit_t(request): + MaxVitT.test(request) # noqa: F405 + + +def test_mobilevit_v2(request): + MobileViTV2.test(request) # noqa: F405 + + +def test_mobilevit_v1(request): + MobileViTV1.test(request) # noqa: F405 + + +def test_pvt(request): + Pvt.test(request) # noqa: F405 + + +def test_regnet(request): + RegNet.test(request) # noqa: F405 + + +def test_retinanet(request): + RetinaNet.test(request) # noqa: F405 + + +def test_roberta(request): + Roberta.test(request) # noqa: F405 + + +def test_squeezenet(request): + SqueezeNet.test(request) # noqa: F405 + + +def test_ssd300_vgg16(request): + Ssd300Vgg16.test(request) # noqa: F405 + + +def test_swin_transformer(request): + SwinTransformer.test(request) # noqa: F405 + + +def test_swin_v2_t(request): + SwinV2T.test(request) # noqa: F405 + + +def test_t5(request): + T5.test(request) # noqa: F405 + + +def test_vit_b_16(request): + VitB16.test(request) # noqa: F405 + + +def test_whisper(request): + Whisper.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/oss/v75/test.py b/backends/qualcomm/tests/rework/htp/e2e/oss/v75/test.py index b5f86874fd4..5f3b3438f36 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/oss/v75/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/oss/v75/test.py @@ -3,3 +3,189 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + +import pytest + +from executorch.backends.qualcomm.tests.rework.src.oss import * # noqa: F403 + + +# -- TestExampleScript (examples/qualcomm/scripts/) -- + + +def test_mobilenet_v2(request): + MobileNetV2.test(request) # noqa: F405 + + +def test_mobilenet_v3(request): + MobileNetV3.test(request) # noqa: F405 + + +def test_inception_v3(request): + InceptionV3.test(request) # noqa: F405 + + +def test_inception_v4(request): + InceptionV4.test(request) # noqa: F405 + + +def test_vit(request): + Vit.test(request) # noqa: F405 + + +def test_edsr(request): + Edsr.test(request) # noqa: F405 + + +def test_deeplab_v3(request): + DeepLabV3.test(request) # noqa: F405 + + +@pytest.mark.skip("dynamic shape inputs appear in recent torch.export.export") +def test_mobilebert(request): + MobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("eagar mode fake quant works well, need further investigation") +def test_ptq_mobilebert(request): + PtqMobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("encountered undefined symbol in mainline, reopen once resolved") +def test_wav2letter(request): + Wav2Letter.test(request) # noqa: F405 + + +def test_export_example(request): + ExportExample.test(request) # noqa: F405 + + +# -- TestExampleOssScript (examples/qualcomm/oss_scripts/) -- + + +def test_albert(request): + Albert.test(request) # noqa: F405 + + +def test_bert(request): + Bert.test(request) # noqa: F405 + + +def test_conv_former(request): + ConvFormer.test(request) # noqa: F405 + + +def test_convnext_small(request): + ConvNextSmall.test(request) # noqa: F405 + + +def test_cvt(request): + Cvt.test(request) # noqa: F405 + + +def test_deit(request): + Deit.test(request) # noqa: F405 + + +def test_depthanything_v2_small(request): + DepthAnythingV2Small.test(request) # noqa: F405 + + +def test_dino_v2(request): + DinoV2.test(request) # noqa: F405 + + +def test_distilbert(request): + Distilbert.test(request) # noqa: F405 + + +def test_dit(request): + Dit.test(request) # noqa: F405 + + +def test_efficientnet(request): + EfficientNet.test(request) # noqa: F405 + + +def test_efficientSAM(request): + EfficientSAM.test(request) # noqa: F405 + + +def test_esrgan(request): + Esrgan.test(request) # noqa: F405 + + +@pytest.mark.skip("Bad accuracy on source model since transformers v5") +def test_eurobert(request): + Eurobert.test(request) # noqa: F405 + + +def test_fastvit(request): + FastVit.test(request) # noqa: F405 + + +def test_fbnet(request): + FbNet.test(request) # noqa: F405 + + +def test_focalnet(request): + FocalNet.test(request) # noqa: F405 + + +def test_gMLP(request): + GMLP.test(request) # noqa: F405 + + +def test_maxvit_t(request): + MaxVitT.test(request) # noqa: F405 + + +def test_mobilevit_v2(request): + MobileViTV2.test(request) # noqa: F405 + + +def test_mobilevit_v1(request): + MobileViTV1.test(request) # noqa: F405 + + +def test_pvt(request): + Pvt.test(request) # noqa: F405 + + +def test_regnet(request): + RegNet.test(request) # noqa: F405 + + +def test_retinanet(request): + RetinaNet.test(request) # noqa: F405 + + +def test_roberta(request): + Roberta.test(request) # noqa: F405 + + +def test_squeezenet(request): + SqueezeNet.test(request) # noqa: F405 + + +def test_ssd300_vgg16(request): + Ssd300Vgg16.test(request) # noqa: F405 + + +def test_swin_transformer(request): + SwinTransformer.test(request) # noqa: F405 + + +def test_swin_v2_t(request): + SwinV2T.test(request) # noqa: F405 + + +def test_t5(request): + T5.test(request) # noqa: F405 + + +def test_vit_b_16(request): + VitB16.test(request) # noqa: F405 + + +def test_whisper(request): + Whisper.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/oss/v79/test.py b/backends/qualcomm/tests/rework/htp/e2e/oss/v79/test.py index b5f86874fd4..5f3b3438f36 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/oss/v79/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/oss/v79/test.py @@ -3,3 +3,189 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + +import pytest + +from executorch.backends.qualcomm.tests.rework.src.oss import * # noqa: F403 + + +# -- TestExampleScript (examples/qualcomm/scripts/) -- + + +def test_mobilenet_v2(request): + MobileNetV2.test(request) # noqa: F405 + + +def test_mobilenet_v3(request): + MobileNetV3.test(request) # noqa: F405 + + +def test_inception_v3(request): + InceptionV3.test(request) # noqa: F405 + + +def test_inception_v4(request): + InceptionV4.test(request) # noqa: F405 + + +def test_vit(request): + Vit.test(request) # noqa: F405 + + +def test_edsr(request): + Edsr.test(request) # noqa: F405 + + +def test_deeplab_v3(request): + DeepLabV3.test(request) # noqa: F405 + + +@pytest.mark.skip("dynamic shape inputs appear in recent torch.export.export") +def test_mobilebert(request): + MobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("eagar mode fake quant works well, need further investigation") +def test_ptq_mobilebert(request): + PtqMobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("encountered undefined symbol in mainline, reopen once resolved") +def test_wav2letter(request): + Wav2Letter.test(request) # noqa: F405 + + +def test_export_example(request): + ExportExample.test(request) # noqa: F405 + + +# -- TestExampleOssScript (examples/qualcomm/oss_scripts/) -- + + +def test_albert(request): + Albert.test(request) # noqa: F405 + + +def test_bert(request): + Bert.test(request) # noqa: F405 + + +def test_conv_former(request): + ConvFormer.test(request) # noqa: F405 + + +def test_convnext_small(request): + ConvNextSmall.test(request) # noqa: F405 + + +def test_cvt(request): + Cvt.test(request) # noqa: F405 + + +def test_deit(request): + Deit.test(request) # noqa: F405 + + +def test_depthanything_v2_small(request): + DepthAnythingV2Small.test(request) # noqa: F405 + + +def test_dino_v2(request): + DinoV2.test(request) # noqa: F405 + + +def test_distilbert(request): + Distilbert.test(request) # noqa: F405 + + +def test_dit(request): + Dit.test(request) # noqa: F405 + + +def test_efficientnet(request): + EfficientNet.test(request) # noqa: F405 + + +def test_efficientSAM(request): + EfficientSAM.test(request) # noqa: F405 + + +def test_esrgan(request): + Esrgan.test(request) # noqa: F405 + + +@pytest.mark.skip("Bad accuracy on source model since transformers v5") +def test_eurobert(request): + Eurobert.test(request) # noqa: F405 + + +def test_fastvit(request): + FastVit.test(request) # noqa: F405 + + +def test_fbnet(request): + FbNet.test(request) # noqa: F405 + + +def test_focalnet(request): + FocalNet.test(request) # noqa: F405 + + +def test_gMLP(request): + GMLP.test(request) # noqa: F405 + + +def test_maxvit_t(request): + MaxVitT.test(request) # noqa: F405 + + +def test_mobilevit_v2(request): + MobileViTV2.test(request) # noqa: F405 + + +def test_mobilevit_v1(request): + MobileViTV1.test(request) # noqa: F405 + + +def test_pvt(request): + Pvt.test(request) # noqa: F405 + + +def test_regnet(request): + RegNet.test(request) # noqa: F405 + + +def test_retinanet(request): + RetinaNet.test(request) # noqa: F405 + + +def test_roberta(request): + Roberta.test(request) # noqa: F405 + + +def test_squeezenet(request): + SqueezeNet.test(request) # noqa: F405 + + +def test_ssd300_vgg16(request): + Ssd300Vgg16.test(request) # noqa: F405 + + +def test_swin_transformer(request): + SwinTransformer.test(request) # noqa: F405 + + +def test_swin_v2_t(request): + SwinV2T.test(request) # noqa: F405 + + +def test_t5(request): + T5.test(request) # noqa: F405 + + +def test_vit_b_16(request): + VitB16.test(request) # noqa: F405 + + +def test_whisper(request): + Whisper.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/htp/e2e/oss/v81/test.py b/backends/qualcomm/tests/rework/htp/e2e/oss/v81/test.py index b5f86874fd4..5f3b3438f36 100644 --- a/backends/qualcomm/tests/rework/htp/e2e/oss/v81/test.py +++ b/backends/qualcomm/tests/rework/htp/e2e/oss/v81/test.py @@ -3,3 +3,189 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + +import pytest + +from executorch.backends.qualcomm.tests.rework.src.oss import * # noqa: F403 + + +# -- TestExampleScript (examples/qualcomm/scripts/) -- + + +def test_mobilenet_v2(request): + MobileNetV2.test(request) # noqa: F405 + + +def test_mobilenet_v3(request): + MobileNetV3.test(request) # noqa: F405 + + +def test_inception_v3(request): + InceptionV3.test(request) # noqa: F405 + + +def test_inception_v4(request): + InceptionV4.test(request) # noqa: F405 + + +def test_vit(request): + Vit.test(request) # noqa: F405 + + +def test_edsr(request): + Edsr.test(request) # noqa: F405 + + +def test_deeplab_v3(request): + DeepLabV3.test(request) # noqa: F405 + + +@pytest.mark.skip("dynamic shape inputs appear in recent torch.export.export") +def test_mobilebert(request): + MobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("eagar mode fake quant works well, need further investigation") +def test_ptq_mobilebert(request): + PtqMobileBert.test(request) # noqa: F405 + + +@pytest.mark.skip("encountered undefined symbol in mainline, reopen once resolved") +def test_wav2letter(request): + Wav2Letter.test(request) # noqa: F405 + + +def test_export_example(request): + ExportExample.test(request) # noqa: F405 + + +# -- TestExampleOssScript (examples/qualcomm/oss_scripts/) -- + + +def test_albert(request): + Albert.test(request) # noqa: F405 + + +def test_bert(request): + Bert.test(request) # noqa: F405 + + +def test_conv_former(request): + ConvFormer.test(request) # noqa: F405 + + +def test_convnext_small(request): + ConvNextSmall.test(request) # noqa: F405 + + +def test_cvt(request): + Cvt.test(request) # noqa: F405 + + +def test_deit(request): + Deit.test(request) # noqa: F405 + + +def test_depthanything_v2_small(request): + DepthAnythingV2Small.test(request) # noqa: F405 + + +def test_dino_v2(request): + DinoV2.test(request) # noqa: F405 + + +def test_distilbert(request): + Distilbert.test(request) # noqa: F405 + + +def test_dit(request): + Dit.test(request) # noqa: F405 + + +def test_efficientnet(request): + EfficientNet.test(request) # noqa: F405 + + +def test_efficientSAM(request): + EfficientSAM.test(request) # noqa: F405 + + +def test_esrgan(request): + Esrgan.test(request) # noqa: F405 + + +@pytest.mark.skip("Bad accuracy on source model since transformers v5") +def test_eurobert(request): + Eurobert.test(request) # noqa: F405 + + +def test_fastvit(request): + FastVit.test(request) # noqa: F405 + + +def test_fbnet(request): + FbNet.test(request) # noqa: F405 + + +def test_focalnet(request): + FocalNet.test(request) # noqa: F405 + + +def test_gMLP(request): + GMLP.test(request) # noqa: F405 + + +def test_maxvit_t(request): + MaxVitT.test(request) # noqa: F405 + + +def test_mobilevit_v2(request): + MobileViTV2.test(request) # noqa: F405 + + +def test_mobilevit_v1(request): + MobileViTV1.test(request) # noqa: F405 + + +def test_pvt(request): + Pvt.test(request) # noqa: F405 + + +def test_regnet(request): + RegNet.test(request) # noqa: F405 + + +def test_retinanet(request): + RetinaNet.test(request) # noqa: F405 + + +def test_roberta(request): + Roberta.test(request) # noqa: F405 + + +def test_squeezenet(request): + SqueezeNet.test(request) # noqa: F405 + + +def test_ssd300_vgg16(request): + Ssd300Vgg16.test(request) # noqa: F405 + + +def test_swin_transformer(request): + SwinTransformer.test(request) # noqa: F405 + + +def test_swin_v2_t(request): + SwinV2T.test(request) # noqa: F405 + + +def test_t5(request): + T5.test(request) # noqa: F405 + + +def test_vit_b_16(request): + VitB16.test(request) # noqa: F405 + + +def test_whisper(request): + Whisper.test(request) # noqa: F405 diff --git a/backends/qualcomm/tests/rework/src/e2e_metrics.py b/backends/qualcomm/tests/rework/src/e2e_metrics.py new file mode 100644 index 00000000000..be54744233b --- /dev/null +++ b/backends/qualcomm/tests/rework/src/e2e_metrics.py @@ -0,0 +1,270 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import operator +from collections.abc import Callable, Mapping +from dataclasses import dataclass + +from executorch.backends.qualcomm.serialization.qc_schema import ( + _soc_info_table, + HtpArch, + LpaiHardwareVersion, + QcomChipset, + QnnExecuTorchBackendType, +) + + +@dataclass(frozen=True) +class ExecutionTarget: + backend: QnnExecuTorchBackendType + hardware_version: HtpArch | LpaiHardwareVersion | None + + +@dataclass(frozen=True) +class MetricConstraint: + operator: Callable[[float, float], bool] + expected: float + + def assert_value(self, actual: float, context: str) -> None: + operator_name = {operator.ge: ">=", operator.le: "<="}.get( + self.operator, self.operator.__name__ + ) + assert self.operator( + actual, self.expected + ), f"{context}: expected {operator_name} {self.expected}, got {actual}" + + +def minimum(value: float) -> MetricConstraint: + return MetricConstraint(operator.ge, value) + + +def maximum(value: float) -> MetricConstraint: + return MetricConstraint(operator.le, value) + + +@dataclass(frozen=True) +class MetricProfile: + shared: Mapping[str, MetricConstraint] + by_backend: Mapping[QnnExecuTorchBackendType, Mapping[str, MetricConstraint]] + by_target: Mapping[ExecutionTarget, Mapping[str, MetricConstraint]] + + +def _profile( + shared: Mapping[str, MetricConstraint], + by_backend: ( + Mapping[QnnExecuTorchBackendType, Mapping[str, MetricConstraint]] | None + ) = None, + by_target: Mapping[ExecutionTarget, Mapping[str, MetricConstraint]] | None = None, +) -> MetricProfile: + return MetricProfile(shared, by_backend or {}, by_target or {}) + + +# TODO: extend targets for coverage +HTP_V75 = ExecutionTarget(QnnExecuTorchBackendType.kHtpBackend, HtpArch.V75) +HTP_V79 = ExecutionTarget(QnnExecuTorchBackendType.kHtpBackend, HtpArch.V79) + + +# TODO: use by_target for fine-grained validation +METRIC_PROFILES = { + "mobilenet_v2": _profile({"top_1": minimum(52), "top_5": minimum(84)}), + "mobilenet_v3": _profile({"top_1": minimum(51), "top_5": minimum(76)}), + "inception_v3": _profile({"top_1": minimum(59), "top_5": minimum(78)}), + "inception_v4": _profile({"top_1": minimum(64), "top_5": minimum(85)}), + "vit": _profile({"top_1": minimum(69), "top_5": minimum(91)}), + "edsr": _profile({"PSNR": minimum(29), "SSIM": minimum(0.94)}), + "deeplab_v3": _profile( + {"PA": minimum(0.88), "MPA": minimum(0.79), "MIoU": minimum(0.67)} + ), + "mobilebert": _profile({"cpu_htp_delta": maximum(2)}), + "ptq_mobilebert": _profile({"cpu_htp_delta": maximum(5)}), + "wav2letter": _profile({"wer": maximum(0.5), "cer": maximum(0.25)}), + "albert": _profile({"accuracy": minimum(0.95)}), + "bert": _profile({"accuracy": minimum(0.55)}), + "conv_former": _profile({"top_1": minimum(70), "top_5": minimum(92)}), + "convnext_small": _profile({"top_1": minimum(74), "top_5": minimum(96)}), + "cvt": _profile({"top_1": minimum(70), "top_5": minimum(90)}), + "deit": _profile({"top_1": minimum(76), "top_5": minimum(92)}), + "depthanything_v2_small": _profile({"sqnr": minimum(15)}), + "dino_v2": _profile({"top_1": minimum(62), "top_5": minimum(87)}), + "distilbert": _profile({"accuracy": minimum(0.48)}), + "dit": _profile({"top_1": minimum(78), "top_5": minimum(92)}), + "efficientnet": _profile({"top_1": minimum(61), "top_5": minimum(88)}), + "efficient_sam": _profile({"MIoU": minimum(0.95)}), + "esrgan": _profile({"PSNR": minimum(23), "SSIM": minimum(0.85)}), + "eurobert": _profile({"accuracy": minimum(0.54)}), + "fastvit": _profile({"top_1": minimum(60), "top_5": minimum(77)}), + "fbnet": _profile({"top_1": minimum(63), "top_5": minimum(88)}), + "focalnet": _profile({"top_1": minimum(54), "top_5": minimum(80)}), + "gmlp": _profile({"top_1": minimum(70), "top_5": minimum(88)}), + "maxvit_t": _profile({"top_1": minimum(71), "top_5": minimum(91)}), + "mobilevit_v2": _profile({"top_1": minimum(50), "top_5": minimum(85)}), + "mobilevit_v1": _profile({"top_1": minimum(76), "top_5": minimum(93)}), + "pvt": _profile({"top_1": minimum(65), "top_5": minimum(83)}), + "regnet": _profile({"top_1": minimum(64), "top_5": minimum(86)}), + "retinanet": _profile({"mAP": minimum(0.6)}), + "roberta": _profile({"accuracy": minimum(0.54)}), + "squeezenet": _profile({"top_1": minimum(27), "top_5": minimum(59)}), + "ssd300_vgg16": _profile({"mAP": minimum(0.76)}), + "swin_transformer": _profile({"top_1": minimum(71), "top_5": minimum(90)}), + "swin_v2_t": _profile({"top_1": minimum(63), "top_5": minimum(91)}), + "t5": _profile({"f1": minimum(0.72)}), + "vit_b_16": _profile({"top_1": minimum(72), "top_5": minimum(96)}), + "whisper": _profile({"wer": maximum(0.25)}), + "codegen2": _profile( + {"pte_size": maximum(1_200_000_000), "inference_speed": minimum(60)} + ), + "llama_stories_260k": _profile( + {"pte_size": maximum(2_020_000), "inference_speed": minimum(1600)} + ), + "llama_stories_110m:int8": _profile( + {"pte_size": maximum(135_000_000), "inference_speed": minimum(220)} + ), + "llama_stories_110m:fp16": _profile( + {"pte_size": maximum(275_000_000), "inference_speed": minimum(220)} + ), + "attention_sink": _profile({"attention_sink_evictor_pte_size": maximum(1_700_000)}), + "asr:granite_speech_3_3-2b": _profile( + { + "audio_encoder_pte_size": maximum(900_000_000), + "tok_embedding_pte_size": maximum(240_000_000), + "pte_size": maximum(3_000_000_000), + }, + by_target={ + HTP_V75: {"inference_speed": minimum(5)}, + HTP_V79: {"inference_speed": minimum(8)}, + }, + ), + "vlm:smolvlm_500m_instruct": _profile( + { + "vision_encoder_pte_size": maximum(110_000_000), + "tok_embedding_pte_size": maximum(100_000_000), + "pte_size": maximum(400_000_000), + }, + by_target={ + HTP_V75: {"inference_speed": minimum(50)}, + HTP_V79: {"inference_speed": minimum(55)}, + }, + ), + "vlm:internvl3_1b": _profile( + { + "vision_encoder_pte_size": maximum(425_000_000), + "tok_embedding_pte_size": maximum(300_000_000), + "pte_size": maximum(550_000_000), + }, + by_target={ + HTP_V75: {"inference_speed": minimum(11)}, + HTP_V79: {"inference_speed": minimum(13)}, + }, + ), +} + + +@dataclass(frozen=True) +class LlmMetricValues: + v75_inference_speed: float + v79_inference_speed: float + pte_size: float + wiki_ppl: float | None = None + hellaswag_acc_norm: float | None = None + sqnr: float | None = None + + +_LLM_METRICS = { + "gemma-2b": LlmMetricValues(32, 36, 2_700_000_000, wiki_ppl=17, sqnr=27), + "gemma2-2b": LlmMetricValues(32, 36, 2_860_000_000, wiki_ppl=14, sqnr=27), + "gemma3-1b": LlmMetricValues(70, 100, 1_200_000_000, wiki_ppl=23, sqnr=10), + "gemma4-e2b": LlmMetricValues(20, 30, 4_500_000_000, wiki_ppl=120, sqnr=10), + "glm-1_5b": LlmMetricValues(42, 52, 1_100_000_000, wiki_ppl=21, sqnr=14), + "granite_3_3-2b_instruct": LlmMetricValues( + 20, 22, 1_600_000_000, hellaswag_acc_norm=0.2 + ), + "phi_4_mini": LlmMetricValues(14, 19, 4_000_000_000, wiki_ppl=14, sqnr=20), + "llama3_2-1b_instruct": LlmMetricValues( + 37, 45, 1_500_000_000, wiki_ppl=18, sqnr=15 + ), + "llama3_2-3b_instruct": LlmMetricValues( + 21, 26, 2_800_000_000, wiki_ppl=11, sqnr=14 + ), + "qwen2_5-0_5b": LlmMetricValues(115, 155, 600_000_000, wiki_ppl=15, sqnr=8), + "qwen2_5-1_5b": LlmMetricValues(38, 47, 1_500_000_000, wiki_ppl=10, sqnr=10), + "qwen3-0_6b": LlmMetricValues(47, 68, 700_000_000, wiki_ppl=21, sqnr=7), + "qwen3-1_7b": LlmMetricValues(28, 34, 1_800_000_000, wiki_ppl=15, sqnr=12), + "smollm2_135m": LlmMetricValues(214, 260, 210_000_000, wiki_ppl=23, sqnr=19), + "smollm3-3b": LlmMetricValues(23, 28, 2_600_000_000, wiki_ppl=10, sqnr=6), +} + +for _model, _metrics in _LLM_METRICS.items(): + _shared = {"pte_size": maximum(_metrics.pte_size)} + if _metrics.wiki_ppl is not None: + _shared["wiki_ppl"] = maximum(_metrics.wiki_ppl) + if _metrics.hellaswag_acc_norm is not None: + _shared["acc_norm"] = minimum(_metrics.hellaswag_acc_norm) + if _metrics.sqnr is not None: + _shared["sqnr"] = minimum(_metrics.sqnr) + METRIC_PROFILES[f"llm:{_model}"] = _profile( + _shared, + by_target={ + HTP_V75: {"inference_speed": minimum(_metrics.v75_inference_speed)}, + HTP_V79: {"inference_speed": minimum(_metrics.v79_inference_speed)}, + }, + ) + +LLM_MODELS = frozenset(_LLM_METRICS) + + +def resolve_execution_target(qnn_config) -> ExecutionTarget: + backend = qnn_config.backend + if backend == QnnExecuTorchBackendType.kGpuBackend: + return ExecutionTarget(backend, None) + + try: + soc_info = _soc_info_table[QcomChipset[qnn_config.soc_model]] + except KeyError as error: + raise ValueError(f"Unknown SoC model: {qnn_config.soc_model}") from error + + if backend == QnnExecuTorchBackendType.kHtpBackend: + if soc_info.htp_info.htp_arch == HtpArch.NONE: + raise ValueError(f"{qnn_config.soc_model} does not support HTP") + return ExecutionTarget(backend, soc_info.htp_info.htp_arch) + if backend == QnnExecuTorchBackendType.kLpaiBackend: + if ( + soc_info.lpai_info is None + or soc_info.lpai_info.lpai_hardware_version == LpaiHardwareVersion.NONE + ): + raise ValueError(f"{qnn_config.soc_model} does not support LPAI") + return ExecutionTarget(backend, soc_info.lpai_info.lpai_hardware_version) + raise ValueError(f"Unsupported backend: {backend}") + + +def get_metric_constraints(workload: str, qnn_config) -> Mapping[str, MetricConstraint]: + profile = METRIC_PROFILES[workload] + target = resolve_execution_target(qnn_config) + return ( + profile.shared + | profile.by_backend.get(target.backend, {}) + | profile.by_target.get(target, {}) + ) + + +def get_metric_constraint( + workload: str, metric: str, qnn_config +) -> MetricConstraint | None: + return get_metric_constraints(workload, qnn_config).get(metric) + + +def assert_metric(workload: str, metric: str, actual: float, qnn_config) -> None: + constraint = get_metric_constraint(workload, metric, qnn_config) + if constraint is None: + return + target = resolve_execution_target(qnn_config) + constraint.assert_value(actual, f"{workload}.{metric} on {target}") + + +def assert_metrics(workload: str, result: Mapping[str, float], qnn_config) -> None: + for metric, constraint in get_metric_constraints(workload, qnn_config).items(): + assert metric in result, f"{workload}: missing metric {metric}" + target = resolve_execution_target(qnn_config) + constraint.assert_value(result[metric], f"{workload}.{metric} on {target}") diff --git a/backends/qualcomm/tests/rework/src/genai.py b/backends/qualcomm/tests/rework/src/genai.py new file mode 100644 index 00000000000..964f0e944b9 --- /dev/null +++ b/backends/qualcomm/tests/rework/src/genai.py @@ -0,0 +1,835 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import json +import logging +import subprocess +from dataclasses import dataclass, replace +from multiprocessing.connection import Listener +from typing import List + +import pytest + +from executorch.backends.qualcomm.tests.rework.conftest import ( + add_default_cmds, + get_ipc, + require_paths, +) +from executorch.backends.qualcomm.tests.rework.src.e2e_metrics import ( + assert_metric, + get_metric_constraint, + LLM_MODELS, +) + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _require(request, artifact_names: list[str] = None): + """Skip if any artifact is None; fail loudly if the path doesn't exist.""" + # executorch_root + artifact_dir are always required for e2e tests.""" + qnn_config = request.getfixturevalue("qnn_config") + options = { + n: request.config.getoption(n) + for n in ["executorch_root", "artifact_dir"] + + (artifact_names if artifact_names else []) + } + # extend this if the option needs no path validation + keys_to_remove = ["model_name", "static_llm_eval_method", "use_fp16"] + if request.config.getoption("model_name") not in _LLAMA_MODELS: + keys_to_remove.append("llama_artifacts") + require_paths({k: v for k, v in options.items() if k not in keys_to_remove}) + return *options.values(), qnn_config + + +def _run(cmds, qnn_config): + ip, port = get_ipc(qnn_config) + p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL) + with Listener((ip, port)) as listener: + conn = listener.accept() + p.communicate() + return json.loads(conn.recv()) + + +def _check(msg): + if "Error" in msg: + pytest.fail(msg["Error"]) + return msg + + +# --------------------------------------------------------------------------- +# LLM models +# --------------------------------------------------------------------------- + + +_LLAMA_MODELS = {"llama3_2-1b_instruct", "llama3_2-3b_instruct"} + + +# --------------------------------------------------------------------------- +# Multimodality model specs +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class MLLMSpecs: + max_seq_len: int + + +@dataclass(frozen=True) +class ALMSpecs(MLLMSpecs): + audio_path: str + golden_audio_feature: str + + +@dataclass(frozen=True) +class VLMSpecs(MLLMSpecs): + image_path: str + golden_image_feature: str + + +ALM_SPECS = { + "granite_speech_3_3-2b": ALMSpecs( + max_seq_len=1024, + audio_path="https://huggingface.co/ibm-granite/granite-speech-3.3-2b/resolve/main/10226_10111_000000.wav?download=true", + golden_audio_feature="after his nap,", + ), +} + +VLM_SPECS = { + "smolvlm_500m_instruct": VLMSpecs( + max_seq_len=1024, + image_path="https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg", + golden_image_feature="city", + ), + "internvl3_1b": VLMSpecs( + max_seq_len=1024, + image_path="http://images.cocodataset.org/val2017/000000039769.jpg", + golden_image_feature="cats", + ), +} + + +# --------------------------------------------------------------------------- +# TestExampleLLMScript +# --------------------------------------------------------------------------- + + +class StaticLLM: + @staticmethod + def test(request): # noqa: C901 + ( + root, + artifact, + model_name, + static_llm_eval_method, + llama_artifacts, + qnn_config, + ) = _require( + request, ["model_name", "static_llm_eval_method", "llama_artifacts"] + ) + assert ( + model_name in LLM_MODELS + ), f"Unable to find {model_name} under LLM_MODELS." + + is_llama_model = model_name in _LLAMA_MODELS + if is_llama_model: + if not llama_artifacts: + pytest.skip("missing required arg: --llama_artifacts") + require_paths( + { + "checkpoint": f"{llama_artifacts}/consolidated.00.pth", + "params": f"{llama_artifacts}/params.json", + "tokenizer": f"{llama_artifacts}/tokenizer.model", + } + ) + + prompt = ( + "I would like to learn python, could you teach me with a simple example?" + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--prompt", + prompt, + "--temperature", + "0", + "--decoder_model", + model_name, + "--model_mode", + "kv", + "--max_seq_len", + "1024", + "--max_context_len", + "1024", + ] + + match static_llm_eval_method: + case "wikitext_ppl": + cmds.extend( + [ + "--eval_methods", + "tasks_eval", + "--eval_tasks", + "wikitext", + "--eval_limit", + "1", + "--calib_tasks", + "wikitext", + "--calib_limit", + "1", + ] + ) + case "hellaswag_acc_norm": + cmds.extend( + [ + "--eval_methods", + "tasks_eval", + "--eval_tasks", + "hellaswag", + "--eval_limit", + "10", + "--calib_tasks", + "hellaswag", + "--calib_limit", + "10", + ] + ) + case "sqnr": + cmds.extend( + [ + "--eval_tasks", + "wikitext", + "--eval_limit", + "1", + "--eval_methods", + "sqnr_eval", + "--calib_tasks", + "wikitext", + "--calib_limit", + "1", + ] + ) + case _: + logging.warning( + "No llm eval method chosen. Only generate model output." + ) + cmds.extend(["--calib_tasks", "wikitext", "--calib_limit", "1"]) + + if is_llama_model: + cmds.extend( + [ + "--checkpoint", + f"{llama_artifacts}/consolidated.00.pth", + "--params", + f"{llama_artifacts}/params.json", + "--tokenizer_model", + f"{llama_artifacts}/tokenizer.model", + ] + ) + + if model_name == "gemma4-e2b": + cmds.extend(["--embedding-quantize", "4,32"]) + + add_default_cmds(cmds, qnn_config) + + msg = _check(_run(cmds, qnn_config)) + logging.info(f"Model Name: {model_name}\nTarget Device: {qnn_config.soc_model}") + logging.info(f"Eval Result: {msg}") + + workload = f"llm:{model_name}" + assert_metric(workload, "pte_size", msg["pte_size"], qnn_config) + + if not qnn_config.compile_only: + if static_llm_eval_method: + metric = { + "wikitext_ppl": "wiki_ppl", + "hellaswag_acc_norm": "acc_norm", + "sqnr": "sqnr", + }[static_llm_eval_method] + assert ( + get_metric_constraint(workload, metric, qnn_config) is not None + ), f"{model_name} currently does not support {static_llm_eval_method}." + assert_metric(workload, metric, msg[metric], qnn_config) + + if ( + not qnn_config.enable_x86_64 + and get_metric_constraint(workload, "inference_speed", qnn_config) + is not None + ): + assert_metric( + workload, "inference_speed", msg["inference_speed"], qnn_config + ) + + +class StaticLLMEvalLimit: + @staticmethod + def test(request): + root, artifact, model_name, llama_artifacts, qnn_config = _require( + request, ["model_name", "llama_artifacts"] + ) + assert ( + model_name in LLM_MODELS + ), f"Unable to find {model_name} under LLM_MODELS." + + is_llama_model = model_name in _LLAMA_MODELS + if is_llama_model: + if not llama_artifacts: + pytest.skip("missing required arg: --llama_artifacts") + require_paths( + { + "checkpoint": f"{llama_artifacts}/consolidated.00.pth", + "params": f"{llama_artifacts}/params.json", + "tokenizer": f"{llama_artifacts}/tokenizer.model", + } + ) + + command_config = replace(qnn_config, compile_only=False, pre_gen_pte=None) + + def run_llama(extra_cmds: List[str]): + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--prompt", + "I would like to learn python, could you teach me with a simple example?", + "--temperature", + "0", + "--decoder_model", + model_name, + "--model_mode", + "kv", + "--max_seq_len", + "1024", + "--max_context_len", + "1024", + ] + cmds.extend(extra_cmds) + add_default_cmds(cmds, command_config) + + msg = _check(_run(cmds, qnn_config)) + return msg + + extra_cmds = [ + "--compile_only", + "--calib_tasks", + "wikitext", + "--calib_limit", + "1", + ] + extra_cmds_for_llama = [ + "--checkpoint", + f"{llama_artifacts}/consolidated.00.pth", + "--params", + f"{llama_artifacts}/params.json", + "--tokenizer_model", + f"{llama_artifacts}/tokenizer.model", + ] + if is_llama_model: + extra_cmds.extend(extra_cmds_for_llama) + + if model_name == "gemma4-e2b": + extra_cmds.extend(["--embedding-quantize", "4,32"]) + + run_llama(extra_cmds) + + def eval_ppl(eval_limit: int) -> float: + extra = [ + "--pre_gen_pte", + artifact, + "--eval_methods", + "tasks_eval", + "--eval_tasks", + "wikitext", + "--eval_limit", + str(eval_limit), + ] + if is_llama_model: + extra.extend(extra_cmds_for_llama) + if qnn_config.device: + extra.extend(["--device", qnn_config.device]) + return run_llama(extra)["wiki_ppl"] + + ppl_1 = eval_ppl(1) + ppl_3 = eval_ppl(3) + logging.info(f"wiki_ppl: eval_limit=1: {ppl_1}, eval_limit=3: {ppl_3}") + assert ppl_1 != ppl_3 + + +class Codegen2: + @staticmethod + def test(request): + root, artifact, qnn_config = _require(request) + prompt = "def hello_world():" + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--prompt", + prompt, + "--temperature", + "0", + "--decoder_model", + "codegen2_1b", + "--model_mode", + "kv", + "--max_seq_len", + "128", + "--max_context_len", + "128", + "--calib_tasks", + "wikitext", + "--calib_limit", + "1", + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + golden_start_with = "def hello_world():" + if not qnn_config.compile_only: + assert msg["result"][0].startswith( + golden_start_with + ), f"Expected Output: {golden_start_with}. Actual Output: {msg['result'][0]}" + if not qnn_config.enable_x86_64: + assert_metric("codegen2", "pte_size", msg["pte_size"], qnn_config) + if not qnn_config.compile_only and not qnn_config.enable_x86_64: + assert_metric( + "codegen2", "inference_speed", msg["inference_speed"], qnn_config + ) + + +class LlamaStories260K: + @staticmethod + def test(request): + root, artifact, llama_artifacts, qnn_config = _require( + request, ["llama_artifacts"] + ) + require_paths( + { + "checkpoint": f"{llama_artifacts}/stories260K.pt", + "params": f"{llama_artifacts}/params.json", + "tokenizer_model": f"{llama_artifacts}/tokenizer.model", + "tokenizer_bin": f"{llama_artifacts}/tokenizer.bin", + } + ) + prompt = "Once" + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--checkpoint", + f"{llama_artifacts}/stories260K.pt", + "--params", + f"{llama_artifacts}/params.json", + "--tokenizer_model", + f"{llama_artifacts}/tokenizer.model", + "--tokenizer_bin", + f"{llama_artifacts}/tokenizer.bin", + "--prompt", + prompt, + "--temperature", + "0", + "--decoder_model", + "stories260k", + "--model_mode", + "hybrid", + "--prefill_ar_len", + "32", + "--max_seq_len", + "128", + "--max_context_len", + "128", + "--calib_tasks", + "wikitext", + "--calib_limit", + "1", + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + golden_start_with = "Once upon a time," + if not qnn_config.compile_only: + assert msg["result"][0].startswith( + golden_start_with + ), f"Expected Output: {golden_start_with}. Actual Output: {msg['result'][0]}" + if not qnn_config.enable_x86_64: + assert_metric("llama_stories_260k", "pte_size", msg["pte_size"], qnn_config) + if not qnn_config.compile_only and not qnn_config.enable_x86_64: + assert_metric( + "llama_stories_260k", + "inference_speed", + msg["inference_speed"], + qnn_config, + ) + + +class LlamaStories110M: + @staticmethod + def test(request): + root, artifact, llama_artifacts, use_fp16, qnn_config = _require( + request, ["llama_artifacts", "use_fp16"] + ) + require_paths( + { + "checkpoint": f"{llama_artifacts}/stories110M.pt", + "params": f"{llama_artifacts}/params.json", + "tokenizer_model": f"{llama_artifacts}/tokenizer.model", + "tokenizer_bin": f"{llama_artifacts}/tokenizer.bin", + } + ) + prompt = "Once" + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--checkpoint", + f"{llama_artifacts}/stories110M.pt", + "--params", + f"{llama_artifacts}/params.json", + "--tokenizer_model", + f"{llama_artifacts}/tokenizer.model", + "--tokenizer_bin", + f"{llama_artifacts}/tokenizer.bin", + "--prompt", + prompt, + "--temperature", + "0", + "--decoder_model", + "stories110m", + "--model_mode", + "hybrid", + "--prefill_ar_len", + "32", + "--max_seq_len", + "128", + "--max_context_len", + "128", + "--calib_tasks", + "wikitext", + "--calib_limit", + "1", + ] + if use_fp16: + cmds.append("--use_fp16") + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + golden_start_with = "Once upon a time," + if not qnn_config.compile_only: + assert msg["result"][0].startswith( + golden_start_with + ), f"Expected Output: {golden_start_with}. Actual Output: {msg['result'][0]}" + workload = "llama_stories_110m:fp16" if use_fp16 else "llama_stories_110m:int8" + if not qnn_config.enable_x86_64: + assert_metric(workload, "pte_size", msg["pte_size"], qnn_config) + if ( + not qnn_config.compile_only + and not qnn_config.enable_x86_64 + and get_metric_constraint(workload, "inference_speed", qnn_config) + is not None + ): + assert_metric( + workload, "inference_speed", msg["inference_speed"], qnn_config + ) + + +class AttentionSink: + @staticmethod + def test(request): + root, artifact, qnn_config = _require(request) + model_name = "smollm2_135m" + prompt = ( + "I would like to learn python, could you teach me with a simple example?" + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--prompt", + prompt, + "--temperature", + "0", + "--decoder_model", + model_name, + "--model_mode", + "kv", + "--max_seq_len", + "2048", + "--max_context_len", + "1024", + "--eval_methods", + "tasks_eval", + "--eval_tasks", + "wikitext", + "--eval_limit", + "1", + "--calib_tasks", + "wikitext", + "--calib_limit", + "1", + "--use_attention_sink", + "4,32", + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + if not qnn_config.compile_only: + assert_metric( + "attention_sink", + "attention_sink_evictor_pte_size", + msg["attention_sink_evictor_pte_size"], + qnn_config, + ) + assert_metric(f"llm:{model_name}", "wiki_ppl", msg["wiki_ppl"], qnn_config) + + +class HFCausalLM: + @staticmethod + def test(request): + root, artifact, qnn_config = _require(request) + prompt = "My favourite condiment is " + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/hf_causal_lm.py", + "--prompt", + prompt, + "--decoder_model", + "qwen2_5-0_5b", + "--ptq", + "16a8w", + "--enable_spinquant_r3", + "--max_seq_len", + "128", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + golden_start_with = "My favourite condiment is iced tea." + if not qnn_config.compile_only: + assert msg["result"][0].startswith( + golden_start_with + ), f"Expected Output: '{golden_start_with}' Actual Output: '{msg['result'][0]}'" + + +class StaticLLMQAT: + @staticmethod + def test(request): + root, artifact, qnn_config = _require(request) + if qnn_config.compile_only: + pytest.skip("tasks_eval requires on-device inference") + + def run_eval( + calib_limit: int, train_limit: int, extra_args: List[str] = None + ) -> float: + prompt = "I would like to learn python, could you teach me with a simple example?" + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--prompt", + prompt, + "--temperature", + "0", + "--decoder_model", + "smollm2_135m", + "--model_mode", + "kv", + "--max_seq_len", + "1024", + "--max_context_len", + "1024", + "--eval_methods", + "tasks_eval", + "--eval_tasks", + "wikitext", + "--eval_limit", + "1", + "--qat", + "--calib_tasks", + "wikitext", + "--calib_limit", + str(calib_limit), + "--train_tasks", + "wikitext", + "--train_limit", + str(train_limit), + ] + if extra_args: + cmds.extend(extra_args) + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + return msg["wiki_ppl"] + + ptq_ppl = run_eval( + calib_limit=1, train_limit=1, extra_args=["--freeze_all_params"] + ) + qat_ppl = run_eval(calib_limit=1, train_limit=1) + logging.info(f"QAT PPL={qat_ppl:.2f}") + logging.info(f"PTQ PPL={ptq_ppl:.2f}") + assert ( + qat_ppl < ptq_ppl + ), f"Expected QAT PPL ({qat_ppl:.2f}) < PTQ PPL({ptq_ppl:.2f})" + + +# --------------------------------------------------------------------------- +# TestExampleMultimodalityScript +# --------------------------------------------------------------------------- + + +class StaticASR: + @staticmethod + def test(request): + root, artifact, model_name, qnn_config = _require(request, ["model_name"]) + + if qnn_config.enable_x86_64: + pytest.skip( + "Skipping the check for the static ASR model on x86 due to long execution time." + ) + + alm_specs: ALMSpecs = ALM_SPECS[model_name] + prompt = "can you transcribe the speech into a written format?" + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--prompt", + prompt, + "--audio_path", + alm_specs.audio_path, + "--temperature", + "0", + "--decoder_model", + model_name, + "--model_mode", + "kv", + "--max_seq_len", + str(alm_specs.max_seq_len), + "--calib_samples", + "./examples/qualcomm/oss_scripts/llama/assets/samples/audio.json", + ] + add_default_cmds(cmds, qnn_config) + + msg = _check(_run(cmds, qnn_config)) + + if not qnn_config.compile_only: + model_out = msg["result"][0] + assert alm_specs.golden_audio_feature in model_out.lower(), ( + f"Expected Output contains feature: '{alm_specs.golden_audio_feature}' " + f"Actual Output: '{model_out}'" + ) + + workload = f"asr:{model_name}" + assert_metric( + workload, + "audio_encoder_pte_size", + msg["audio_encoder_pte_size"], + qnn_config, + ) + assert_metric( + workload, + "tok_embedding_pte_size", + msg["tok_embedding_pte_size"], + qnn_config, + ) + assert_metric(workload, "pte_size", msg["pte_size"], qnn_config) + if ( + not qnn_config.compile_only + and get_metric_constraint(workload, "inference_speed", qnn_config) + is not None + ): + assert_metric( + workload, "inference_speed", msg["inference_speed"], qnn_config + ) + + +class StaticVLM: + @staticmethod + def test(request): + root, artifact, model_name, qnn_config = _require(request, ["model_name"]) + + vlm_specs: VLMSpecs = VLM_SPECS[model_name] + prompt = "Can you describe this image?" + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/llama/llama.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--prompt", + prompt, + "--image_path", + vlm_specs.image_path, + "--temperature", + "0", + "--decoder_model", + model_name, + "--model_mode", + "kv", + "--max_seq_len", + str(vlm_specs.max_seq_len), + "--calib_samples", + "./examples/qualcomm/oss_scripts/llama/assets/samples/vision.json", + ] + add_default_cmds(cmds, qnn_config) + + msg = _check(_run(cmds, qnn_config)) + + if not qnn_config.compile_only: + model_out = msg["result"][0] + assert vlm_specs.golden_image_feature in model_out, ( + f"Expected Output contains feature: '{vlm_specs.golden_image_feature}' " + f"Actual Output: '{model_out}'" + ) + + workload = f"vlm:{model_name}" + if not qnn_config.enable_x86_64: + assert_metric( + workload, + "vision_encoder_pte_size", + msg["vision_encoder_pte_size"], + qnn_config, + ) + assert_metric( + workload, + "tok_embedding_pte_size", + msg["tok_embedding_pte_size"], + qnn_config, + ) + assert_metric(workload, "pte_size", msg["pte_size"], qnn_config) + if ( + not qnn_config.compile_only + and not qnn_config.enable_x86_64 + and get_metric_constraint(workload, "inference_speed", qnn_config) + is not None + ): + assert_metric( + workload, "inference_speed", msg["inference_speed"], qnn_config + ) diff --git a/backends/qualcomm/tests/rework/src/oss.py b/backends/qualcomm/tests/rework/src/oss.py new file mode 100644 index 00000000000..fc9c9755d0e --- /dev/null +++ b/backends/qualcomm/tests/rework/src/oss.py @@ -0,0 +1,931 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import json +import subprocess +import tempfile +from multiprocessing.connection import Listener +from pathlib import Path + +import pytest + +from executorch.backends.qualcomm.serialization.qc_schema import ( + QnnExecuTorchBackendType, +) +from executorch.backends.qualcomm.tests.rework.conftest import ( + add_default_cmds, + get_ipc, + require_paths, +) +from executorch.backends.qualcomm.tests.rework.src.e2e_metrics import ( + assert_metric, + assert_metrics, +) + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _require(request, artifact_names: list[str] = None): + """Skip if any artifact is None; fail loudly if the path doesn't exist.""" + # executorch_root + artifact_dir are always required for e2e tests.""" + qnn_config = request.getfixturevalue("qnn_config") + options = { + n: request.config.getoption(n) + for n in ["executorch_root", "artifact_dir"] + + (artifact_names if artifact_names else []) + } + # extend this if the option needs no path validation + keys_to_remove = ["model_name"] + require_paths({k: v for k, v in options.items() if k not in keys_to_remove}) + return *options.values(), qnn_config + + +def _run(cmds, qnn_config): + ip, port = get_ipc(qnn_config) + p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL) + with Listener((ip, port)) as listener: + conn = listener.accept() + p.communicate() + return json.loads(conn.recv()) + + +def _check(msg): + if "Error" in msg: + pytest.fail(msg["Error"]) + return msg + + +# --------------------------------------------------------------------------- +# TestExampleScript (examples/qualcomm/scripts/) +# --------------------------------------------------------------------------- + + +class MobileNetV2: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/mobilenet_v2.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("mobilenet_v2", msg, qnn_config) + + +class MobileNetV3: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/mobilenet_v3.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("mobilenet_v3", msg, qnn_config) + + +class InceptionV3: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/inception_v3.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("inception_v3", msg, qnn_config) + + +class InceptionV4: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/inception_v4.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("inception_v4", msg, qnn_config) + + +class Vit: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/torchvision_vit.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("vit", msg, qnn_config) + + +class Edsr: + @staticmethod + def test(request): + root, artifact, qnn_config = _require(request) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/edsr.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--default_dataset", + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("edsr", msg, qnn_config) + + +class DeepLabV3: + @staticmethod + def test(request): + root, artifact, qnn_config = _require(request) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/deeplab_v3.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("deeplab_v3", msg, qnn_config) + + +class MobileBert: + @staticmethod + def test(request): + root, artifact, pretrained_weight, qnn_config = _require( + request, ["pretrained_weight"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/mobilebert_fine_tune.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--pretrained_weight", + pretrained_weight, + "--use_fp16", + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + cpu, htp = msg["CPU"], msg["HTP"] + for k, v in cpu.items(): + assert_metric( + "mobilebert", "cpu_htp_delta", abs(v[0] - htp[k][0]), qnn_config + ) + + +class PtqMobileBert: + @staticmethod + def test(request): + root, artifact, pretrained_weight, qnn_config = _require( + request, ["pretrained_weight"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/mobilebert_fine_tune.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--pretrained_weight", + pretrained_weight, + "--ptq", + "16a16w", + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + cpu, htp = msg["CPU"], msg["HTP"] + for k, v in cpu.items(): + assert_metric( + "ptq_mobilebert", "cpu_htp_delta", abs(v[0] - htp[k][0]), qnn_config + ) + + +class Wav2Letter: + @staticmethod + def test(request): + root, artifact, pretrained_weight, qnn_config = _require( + request, ["pretrained_weight"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/scripts/wav2letter.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--pretrained_weight", + pretrained_weight, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("wav2letter", msg, qnn_config) + + +class ExportExample: + @staticmethod + def test(request): + root, artifact, model_name, qnn_config = _require(request, ["model_name"]) + with tempfile.TemporaryDirectory() as tmp_dir: + cmds = [ + "python", + "qualcomm/scripts/export_example.py", + "--model_name", + model_name, + "--output_folder", + f"{tmp_dir}/", + "--generate_etrecord", + ] + p = subprocess.Popen( + cmds, + stdout=subprocess.DEVNULL, + cwd=f"{root}/examples", + ) + p.communicate() + assert Path(f"{tmp_dir}/{model_name}.pte").exists() + + +# --------------------------------------------------------------------------- +# TestExampleOssScript (examples/qualcomm/oss_scripts/) +# --------------------------------------------------------------------------- + + +class Albert: + @staticmethod + def test(request): + root, artifact, sentence_dataset, qnn_config = _require( + request, ["sentence_dataset"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/albert.py", + "--dataset", + sentence_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("albert", msg, qnn_config) + + +class Bert: + @staticmethod + def test(request): + root, artifact, sentence_dataset, qnn_config = _require( + request, ["sentence_dataset"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/bert.py", + "--dataset", + sentence_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("bert", msg, qnn_config) + + +class ConvFormer: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/conv_former.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("conv_former", msg, qnn_config) + + +class ConvNextSmall: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/convnext_small.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("convnext_small", msg, qnn_config) + + +class Cvt: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/cvt.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("cvt", msg, qnn_config) + + +class Deit: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/deit.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("deit", msg, qnn_config) + + +class DepthAnythingV2Small: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/depthanything_v2_small.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("depthanything_v2_small", msg, qnn_config) + + +class DinoV2: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/dino_v2.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("dino_v2", msg, qnn_config) + + +class Distilbert: + @staticmethod + def test(request): + root, artifact, sentence_dataset, qnn_config = _require( + request, ["sentence_dataset"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/distilbert.py", + "--dataset", + sentence_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("distilbert", msg, qnn_config) + + +class Dit: + @staticmethod + def test(request): + root, artifact, qnn_config = _require(request) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/dit.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("dit", msg, qnn_config) + + +class EfficientNet: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/efficientnet.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("efficientnet", msg, qnn_config) + + +class EfficientSAM: + @staticmethod + def test(request): + root, artifact, image_dataset, pretrained_weight, oss_repo, qnn_config = ( + _require(request, ["image_dataset", "pretrained_weight", "oss_repo"]) + ) + if qnn_config.backend == QnnExecuTorchBackendType.kHtpBackend: + pytest.skip("Bad accuracy, need investigation") + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/efficientSAM/efficientSAM.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--oss_repo", + oss_repo, + "--pretrained_weight", + pretrained_weight, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("efficient_sam", msg, qnn_config) + + +class Esrgan: + @staticmethod + def test(request): + root, artifact, oss_repo, qnn_config = _require(request, ["oss_repo"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/esrgan.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--default_dataset", + "--oss_repo", + oss_repo, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("esrgan", msg, qnn_config) + + +class Eurobert: + @staticmethod + def test(request): + root, artifact, sentence_dataset, qnn_config = _require( + request, ["sentence_dataset"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/eurobert.py", + "--dataset", + sentence_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("eurobert", msg, qnn_config) + + +class FastVit: + @staticmethod + def test(request): + root, artifact, image_dataset, pretrained_weight, oss_repo, qnn_config = ( + _require(request, ["image_dataset", "pretrained_weight", "oss_repo"]) + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/fastvit.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--oss_repo", + oss_repo, + "--pretrained_weight", + pretrained_weight, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("fastvit", msg, qnn_config) + + +class FbNet: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/fbnet.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("fbnet", msg, qnn_config) + + +class FocalNet: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/focalnet.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("focalnet", msg, qnn_config) + + +class GMLP: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/gMLP_image_classification.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("gmlp", msg, qnn_config) + + +class MaxVitT: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/maxvit_t.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("maxvit_t", msg, qnn_config) + + +class MobileViTV2: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/mobilevit_v2.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("mobilevit_v2", msg, qnn_config) + + +class MobileViTV1: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/mobilevit_v1.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("mobilevit_v1", msg, qnn_config) + + +class Pvt: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/pvt.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("pvt", msg, qnn_config) + + +class RegNet: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/regnet.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + for weight in ["regnet_y_400mf", "regnet_x_400mf"]: + msg = _check(_run(cmds + ["--weights", weight], qnn_config)) + assert_metrics("regnet", msg, qnn_config) + + +class RetinaNet: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/retinanet.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--dataset", + image_dataset, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("retinanet", msg, qnn_config) + + +class Roberta: + @staticmethod + def test(request): + root, artifact, sentence_dataset, qnn_config = _require( + request, ["sentence_dataset"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/roberta.py", + "--dataset", + sentence_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("roberta", msg, qnn_config) + + +class SqueezeNet: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/squeezenet.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("squeezenet", msg, qnn_config) + + +class Ssd300Vgg16: + @staticmethod + def test(request): + root, artifact, pretrained_weight, oss_repo, qnn_config = _require( + request, ["pretrained_weight", "oss_repo"] + ) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/ssd300_vgg16.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + "--oss_repo", + oss_repo, + "--pretrained_weight", + pretrained_weight, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("ssd300_vgg16", msg, qnn_config) + + +class SwinTransformer: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/swin_transformer.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("swin_transformer", msg, qnn_config) + + +class SwinV2T: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/swin_v2_t.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("swin_v2_t", msg, qnn_config) + + +class T5: + @staticmethod + def test(request): + root, artifact, qa_dataset, qnn_config = _require(request, ["qa_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/t5/t5.py", + "--dataset", + qa_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("t5", msg, qnn_config) + + +class VitB16: + @staticmethod + def test(request): + root, artifact, image_dataset, qnn_config = _require(request, ["image_dataset"]) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/vit_b_16.py", + "--dataset", + image_dataset, + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("vit_b_16", msg, qnn_config) + + +class Whisper: + @staticmethod + def test(request): + root, artifact, qnn_config = _require(request) + cmds = [ + "python", + f"{root}/examples/qualcomm/oss_scripts/whisper/whisper.py", + "--artifact", + artifact, + "--build_folder", + qnn_config.build_folder, + ] + add_default_cmds(cmds, qnn_config) + msg = _check(_run(cmds, qnn_config)) + assert_metrics("whisper", msg, qnn_config) diff --git a/backends/qualcomm/tests/test_qnn_delegate.py b/backends/qualcomm/tests/test_qnn_delegate.py index 221b7b4dbac..10770bf58e1 100644 --- a/backends/qualcomm/tests/test_qnn_delegate.py +++ b/backends/qualcomm/tests/test_qnn_delegate.py @@ -9408,7 +9408,6 @@ def run_llama(extra_cmds): "1024", "--max_context_len", "1024", - "--skip_user_prompt_calibration", "--soc_model", self.soc_model, "--target",