diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index e1fa14e66e9..52c009b5608 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -1365,9 +1365,9 @@ jobs: export QNN_DELEGATE_TEST=1 export LD_LIBRARY_PATH="$QNN_SDK_ROOT/lib/x86_64-linux-clang/:$(realpath build-x86/lib/):${LD_LIBRARY_PATH:-}" - # Run operator test classes only. + # Run operator and FCB test classes only. pytest -vs backends/qualcomm/tests/test_qnn_delegate.py \ - -k "TestQNNFloatingPointOperator or TestQNNQuantizedOperator" + -k "TestQNNFloatingPointOperator or TestQNNQuantizedOperator or TestQNNFcb" test-phi-3-mini-runner-linux: needs: docker-image diff --git a/backends/qualcomm/qnn_preprocess.py b/backends/qualcomm/qnn_preprocess.py index 71088ed1ab2..dda4687e59c 100644 --- a/backends/qualcomm/qnn_preprocess.py +++ b/backends/qualcomm/qnn_preprocess.py @@ -158,32 +158,50 @@ def preprocess( ) -> PreprocessResult: option = generate_qnn_executorch_option(compile_specs) obj_options = flatbuffer_to_option(option) + if obj_options.target_options is not None: + return QnnBackend.preprocess_multimethod( + {DEFAULT_GRAPH_NAME: [edge_program]}, + {DEFAULT_GRAPH_NAME: [compile_specs]}, + )[DEFAULT_GRAPH_NAME][0] qnn_manager = get_current_qnn_manager(compile_specs) - qnn_manager.InitContext([DEFAULT_GRAPH_NAME]) - py_op_wrapper_list = QnnBackend._build_op_wrappers( - edge_program, - qnn_manager.IsTensorDump(), - obj_options.op_package_options.op_package_infos, - obj_options.use_mha2sha, - obj_options.backend_options.backend_type, - ) + init_result = qnn_manager.InitContext([DEFAULT_GRAPH_NAME]) + try: + if init_result.value != 0: + raise RuntimeError( + f"Failed to initialize QNN context: {init_result.value}" + ) + py_op_wrapper_list = QnnBackend._build_op_wrappers( + edge_program, + qnn_manager.IsTensorDump(), + obj_options.op_package_options.op_package_infos, + obj_options.use_mha2sha, + obj_options.backend_options.backend_type, + ) - qnn_context_binary = qnn_manager.Compile( - qnn_manager.GetGraphNames(), - [[py_op_wrapper.GetOpWrapper() for py_op_wrapper in py_op_wrapper_list]], - ) + qnn_context_binary = qnn_manager.Compile( + qnn_manager.GetGraphNames(), + [ + [ + py_op_wrapper.GetOpWrapper() + for py_op_wrapper in py_op_wrapper_list + ] + ], + ) - if obj_options.saver: - exit( - f"Record all QNN API calls from saver backend at: {obj_options.saver_output_dir}" + if obj_options.saver: + exit( + f"Record all QNN API calls from saver backend at: {obj_options.saver_output_dir}" + ) + assert ( + len(qnn_context_binary) != 0 + ), "Failed to generate Qnn context binary." + # For now, debug_handle_map is not used by QNN ExecuTorch + return PreprocessResult( + processed_bytes=qnn_context_binary, + debug_handle_map={}, ) - assert len(qnn_context_binary) != 0, "Failed to generate Qnn context binary." - qnn_manager.DestroyContext() - # For now, debug_handle_map is not used by QNN ExecuTorch - return PreprocessResult( - processed_bytes=qnn_context_binary, - debug_handle_map={}, - ) + finally: + qnn_manager.DestroyContext() @staticmethod def _populate_delegate_mapping( @@ -207,8 +225,12 @@ def _populate_delegate_mapping( @staticmethod def _get_compile_func(qnn_manager: PyQnnManager.QnnManager): def compile_func(graph_names, op_wrapper_list): - qnn_manager.InitContext(graph_names) + init_result = qnn_manager.InitContext(graph_names) try: + if init_result.value != 0: + raise RuntimeError( + f"Failed to initialize QNN context: {init_result.value}" + ) qnn_context_binary = qnn_manager.Compile(graph_names, op_wrapper_list) finally: qnn_manager.DestroyContext() @@ -221,8 +243,13 @@ def _get_compile_func_fcb(qnn_managers: List[PyQnnManager.QnnManager]): def compile_func(graph_names, op_wrapper_list): with qnn_managers[0].CreateDlc() as dlc_handle: for qnn_manager in qnn_managers: - qnn_manager.InitContext(graph_names) + init_result = qnn_manager.InitContext(graph_names) try: + if init_result.value != 0: + raise RuntimeError( + "Failed to initialize QNN context: " + f"{init_result.value}" + ) qnn_manager.CompileToDlc( graph_names, op_wrapper_list, dlc_handle ) diff --git a/backends/qualcomm/tests/test_qnn_delegate.py b/backends/qualcomm/tests/test_qnn_delegate.py index 221b7b4dbac..eca38e27451 100644 --- a/backends/qualcomm/tests/test_qnn_delegate.py +++ b/backends/qualcomm/tests/test_qnn_delegate.py @@ -17,7 +17,8 @@ from functools import partial from multiprocessing.connection import Listener from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch +from zipfile import ZipFile import torch from executorch.backends.qualcomm._passes.qnn_pass_manager import ( @@ -35,6 +36,7 @@ make_quantizer, setup_common_args_and_variables, ) +from executorch.backends.qualcomm.qnn_preprocess import DEFAULT_GRAPH_NAME, QnnBackend from executorch.backends.qualcomm.quantizer.rules import Q_ANNOTATION_KEY from executorch.backends.qualcomm.serialization.qc_schema import ( QcomChipset, @@ -67,6 +69,7 @@ QCOM_QUANT_ATTRS_MAP, QCOM_QUANT_DTYPE, QCOM_SAMPLE_INPUTS, + QCOM_TENSOR_NAME, ) from executorch.backends.qualcomm.utils.utils import ( capture_program, @@ -111,7 +114,9 @@ from executorch.examples.models.wav2letter import Wav2LetterModel from executorch.exir import to_edge -from executorch.exir.backend.backend_api import disable_validation +from executorch.exir.backend.backend_api import disable_validation, to_backend +from executorch.exir.backend.backend_details import PreprocessResult +from executorch.exir.debug_handle_utils import DEBUG_HANDLE_KEY from torchao.quantization.pt2e.quantizer import SharedQuantizationSpec @@ -12254,6 +12259,86 @@ def test_fcb_compiler_spec_preserves_targets(self): [target.soc_info.soc_model for target in option.target_options.targets], [QcomChipset.SM8650, QcomChipset.SM8750], ) + self.assertEqual( + [ + target.backend_options[0].htp_options.precision + for target in option.target_options.targets + ], + [option.htp_options.precision for option in options], + ) + self.assertTrue(option.target_options.fcb_reference_weight_sharing) + + def test_fcb_rejects_invalid_target_configuration(self): + htp = generate_htp_compiler_spec(use_fp16=False) + cases = [ + ({"soc_model": [], "backend_options": []}, "must not be empty"), + ( + { + "soc_model": [QcomChipset.SM8650, QcomChipset.SM8750], + "backend_options": [htp], + }, + "equal-length", + ), + ( + { + "soc_model": [QcomChipset.SM8650, QcomChipset.SM8750], + "backend_options": [htp, generate_gpu_compiler_spec()], + }, + "HTP", + ), + ( + { + "soc_model": [QcomChipset.SM8650, QcomChipset.SM8750], + "backend_options": [htp, htp], + "online_prepare": True, + }, + "offline_prepare", + ), + ( + {"soc_model": QcomChipset.SM8650, "backend_options": [htp, htp]}, + "equal-length", + ), + ( + { + "soc_model": [QcomChipset.SM8650, QcomChipset.SM8750], + "backend_options": htp, + }, + "equal-length", + ), + ] + for kwargs, message in cases: + with self.subTest(kwargs=kwargs), self.assertRaisesRegex( + ValueError, message + ): + generate_qnn_executorch_compiler_spec(**kwargs) + + def test_fcb_rejects_dlbc_with_reference_weight_sharing(self): + htp = generate_htp_compiler_spec(use_fp16=False, use_dlbc=True) + with self.assertRaisesRegex(ValueError, "DLBC"): + generate_qnn_executorch_compiler_spec( + soc_model=[QcomChipset.SM8650, QcomChipset.SM8750], + backend_options=[htp, htp], + ) + + def test_fcb_single_target_remains_scalar(self): + htp = generate_htp_compiler_spec(use_fp16=False) + for soc_model, backend_options in ( + (QcomChipset.SM8650, htp), + ([QcomChipset.SM8650], [htp]), + (QcomChipset.SM8650, [htp]), + ([QcomChipset.SM8650], htp), + ): + with self.subTest(soc_model=soc_model, backend_options=backend_options): + option = flatbuffer_to_option( + generate_qnn_executorch_compiler_spec( + soc_model=soc_model, backend_options=backend_options + )[0].value + ) + self.assertIsNone(option.target_options) + self.assertEqual( + option.backend_options.backend_type, + QnnExecuTorchBackendType.kHtpBackend, + ) def test_fcb_manager_cache_is_keyed_by_soc(self): managers = [Mock(), Mock(), Mock()] @@ -12320,6 +12405,284 @@ def test_fcb_reference_weight_sharing_e2e(self): self.verify_output(module, inputs, executorch_program) + def _reset_active_registry(self): + qnn_manager_lifecycle._current_qnn_managers.active_registry = None + self.addCleanup( + setattr, + qnn_manager_lifecycle._current_qnn_managers, + "active_registry", + None, + ) + + def _fcb_specs(self): + htp = generate_htp_compiler_spec(use_fp16=True) + return generate_qnn_executorch_compiler_spec( + soc_model=[QcomChipset.SM8650, QcomChipset.SM8750], + backend_options=[htp, htp], + ) + + def _scalar_specs(self): + return generate_qnn_executorch_compiler_spec( + soc_model=QcomChipset.SM8750, + backend_options=generate_htp_compiler_spec(use_fp16=True), + ) + + def test_fcb_context_creates_each_target_once_and_destroys_them(self): + self._reset_active_registry() + specs = self._fcb_specs() + managers = [Mock(), Mock()] + for manager in managers: + manager.InitBackend.return_value = Mock(value=0) + + with patch.object( + qnn_manager_lifecycle.PyQnnManager, "QnnManager", side_effect=managers + ) as create: + with qnn_manager_lifecycle.QnnManagerContext({"forward": specs}): + first = qnn_manager_lifecycle.get_current_qnn_manager( + specs, QcomChipset.SM8650 + ) + second = qnn_manager_lifecycle.get_current_qnn_manager( + specs, QcomChipset.SM8750 + ) + self.assertIs( + first, + qnn_manager_lifecycle.get_current_qnn_manager( + specs, QcomChipset.SM8650 + ), + ) + + self.assertEqual(create.call_count, 2) + self.assertIs(first, managers[0]) + self.assertIs(second, managers[1]) + for manager in managers: + manager.Destroy.assert_called_once_with() + + def test_fcb_manager_outside_context_is_not_retained(self): + self._reset_active_registry() + specs = self._scalar_specs() + managers = [Mock(), Mock()] + for manager in managers: + manager.InitBackend.return_value = Mock(value=0) + + with patch.object( + qnn_manager_lifecycle.PyQnnManager, "QnnManager", side_effect=managers + ) as create: + first = qnn_manager_lifecycle.get_current_qnn_manager(specs) + second = qnn_manager_lifecycle.get_current_qnn_manager(specs) + + self.assertIsNot(first, second) + self.assertEqual(create.call_count, 2) + self.assertIsNone(qnn_manager_lifecycle._current_qnn_managers.active_registry) + + def test_fcb_context_restores_previous_registry_when_cleanup_fails(self): + self._reset_active_registry() + previous_registry = Mock() + qnn_manager_lifecycle._current_qnn_managers.active_registry = previous_registry + current_registry = Mock() + current_registry.destroy_all.side_effect = RuntimeError("cleanup failed") + + with patch.object( + qnn_manager_lifecycle, "QnnManagerRegistry", return_value=current_registry + ), self.assertRaisesRegex(RuntimeError, "cleanup failed"): + with qnn_manager_lifecycle.QnnManagerContext({}): + self.assertIs( + qnn_manager_lifecycle._current_qnn_managers.active_registry, + current_registry, + ) + + self.assertIs( + qnn_manager_lifecycle._current_qnn_managers.active_registry, + previous_registry, + ) + + def test_fcb_destroy_all_raises_first_failure_and_logs_rest(self): + registry = qnn_manager_lifecycle.QnnManagerRegistry() + managers = [Mock(), Mock(), Mock()] + first_error = RuntimeError("first cleanup failed") + managers[0].Destroy.side_effect = first_error + managers[2].Destroy.side_effect = ValueError("second cleanup failed") + registry._registry = { + "first": managers[0], + "second": managers[1], + "third": managers[2], + } + + with self.assertLogs(level="ERROR") as logs, self.assertRaises( + RuntimeError + ) as raised: + registry.destroy_all() + + self.assertIs(raised.exception, first_error) + self.assertIn("second cleanup failed", logs.output[0]) + for manager in managers: + manager.Destroy.assert_called_once_with() + self.assertEqual(registry._registry, {}) + + def test_fcb_lookup_rejects_unknown_target(self): + with self.assertRaisesRegex(ValueError, "do not target SM8850"): + qnn_manager_lifecycle.get_current_qnn_manager( + self._fcb_specs(), QcomChipset.SM8850 + ) + + def test_fcb_single_program_uses_multimethod_compiler(self): + program = Mock() + specs = self._fcb_specs() + expected = PreprocessResult(processed_bytes=b"fcb", debug_handle_map={}) + + with patch.object( + QnnBackend, + "preprocess_multimethod", + return_value={DEFAULT_GRAPH_NAME: [expected]}, + ) as preprocess_multimethod: + actual = QnnBackend.preprocess(program, specs) + + self.assertIs(actual, expected) + preprocess_multimethod.assert_called_once_with( + {DEFAULT_GRAPH_NAME: [program]}, + {DEFAULT_GRAPH_NAME: [specs]}, + ) + + def test_fcb_to_backend_produces_dlc(self): + def edge_program(): + torch.manual_seed(0) + exported = torch.export.export( + Conv2dSingle().eval(), # noqa: F405 + (torch.ones(1, 1, 16, 16),), + strict=True, + ) + return to_edge(exported).exported_program() + + scalar = to_backend(QnnBackend.__name__, edge_program(), self._scalar_specs()) + fcb = to_backend(QnnBackend.__name__, edge_program(), self._fcb_specs()) + + self.assertGreater(len(scalar.processed_bytes), 0) + dlc_bytes = bytes(PyQnnManagerAdaptor.StripProtocol(fcb.processed_bytes)) + self.assertEqual(dlc_bytes[:4], b"PK\x03\x04") + with ZipFile(io.BytesIO(dlc_bytes)) as dlc: + members = dlc.namelist() + self.assertEqual( + sorted(name for name in members if name.startswith("htp.cache.")), + ["htp.cache.sm8650.0", "htp.cache.sm8750.0"], + ) + self.assertEqual( + sum(name.startswith("htp.shared.weights.") for name in members), 1 + ) + + def _preprocess_fcb_with_managers(self, managers): + op_wrapper = Mock() + op_wrapper.GetOpWrapper.return_value = Mock() + with patch( + "executorch.backends.qualcomm.qnn_preprocess.get_current_qnn_manager", + side_effect=managers, + ), patch.object(QnnBackend, "_build_op_wrappers", return_value=[op_wrapper]): + return QnnBackend.preprocess_multimethod( + {DEFAULT_GRAPH_NAME: [Mock()]}, + {DEFAULT_GRAPH_NAME: [self._fcb_specs()]}, + ) + + def _preprocess_scalar_with_manager(self, manager): + op_wrapper = Mock() + op_wrapper.GetOpWrapper.return_value = Mock() + manager.GetGraphNames.return_value = [DEFAULT_GRAPH_NAME] + with patch( + "executorch.backends.qualcomm.qnn_preprocess.get_current_qnn_manager", + return_value=manager, + ), patch.object(QnnBackend, "_build_op_wrappers", return_value=[op_wrapper]): + return QnnBackend.preprocess(Mock(), self._scalar_specs()) + + def test_fcb_scalar_preprocess_releases_context_after_init_failure(self): + manager = Mock() + manager.InitContext.return_value = Mock(value=1) + + with self.assertRaisesRegex(RuntimeError, "initialize QNN context"): + self._preprocess_scalar_with_manager(manager) + + manager.Compile.assert_not_called() + manager.DestroyContext.assert_called_once_with() + + def test_fcb_scalar_preprocess_releases_context_after_compile_failure(self): + manager = Mock() + manager.InitContext.return_value = Mock(value=0) + manager.Compile.side_effect = RuntimeError("compile failed") + + with self.assertRaisesRegex(RuntimeError, "compile failed"): + self._preprocess_scalar_with_manager(manager) + + manager.DestroyContext.assert_called_once_with() + + def _fcb_managers(self, init_values): + managers = [Mock() for _ in init_values] + for manager, value in zip(managers, init_values): + manager.InitContext.return_value = Mock(value=value) + dlc_context = MagicMock() + dlc_context.__enter__.return_value = Mock() + managers[0].CreateDlc.return_value = dlc_context + managers[0].GetDlcBinary.return_value = b"fcb" + return managers, dlc_context + + def test_fcb_compile_releases_contexts_and_dlc(self): + managers, dlc_context = self._fcb_managers([0, 0]) + + result = self._preprocess_fcb_with_managers(managers) + + self.assertEqual(result[DEFAULT_GRAPH_NAME][0].processed_bytes, b"fcb") + for manager in managers: + manager.CompileToDlc.assert_called_once() + manager.DestroyContext.assert_called_once_with() + dlc_context.__exit__.assert_called_once() + + def test_fcb_compile_releases_resources_after_compile_failure(self): + managers, dlc_context = self._fcb_managers([0, 0]) + managers[1].CompileToDlc.side_effect = RuntimeError("append failed") + + with self.assertRaisesRegex(RuntimeError, "append failed"): + self._preprocess_fcb_with_managers(managers) + + for manager in managers: + manager.DestroyContext.assert_called_once_with() + dlc_context.__exit__.assert_called_once() + + def test_fcb_compile_releases_resources_after_init_failure(self): + managers, dlc_context = self._fcb_managers([0, 1]) + + with self.assertRaisesRegex(RuntimeError, "initialize QNN context"): + self._preprocess_fcb_with_managers(managers) + + managers[0].CompileToDlc.assert_called_once() + managers[1].CompileToDlc.assert_not_called() + for manager in managers: + manager.DestroyContext.assert_called_once_with() + dlc_context.__exit__.assert_called_once() + + def test_fcb_debug_mapping_built_after_wrappers(self): + node = Mock(op="call_function", meta={DEBUG_HANDLE_KEY: 7}) + program = Mock() + program.graph.nodes = [node] + specs = generate_qnn_executorch_compiler_spec( + soc_model=QcomChipset.SM8750, + backend_options=generate_htp_compiler_spec(use_fp16=True), + dump_intermediate_outputs=True, + ) + + def build_op_wrappers(*_args): + node.meta[QCOM_TENSOR_NAME] = ["tensor_0"] + return b"ctx" + + with patch( + "executorch.backends.qualcomm.qnn_preprocess.get_current_qnn_manager", + return_value=Mock(), + ), patch.object( + QnnBackend, "_build_op_wrappers", side_effect=build_op_wrappers + ): + result = QnnBackend.preprocess_multimethod( + {DEFAULT_GRAPH_NAME: [program]}, + {DEFAULT_GRAPH_NAME: [specs]}, + ) + + self.assertEqual( + result[DEFAULT_GRAPH_NAME][0].debug_handle_map, {"tensor_0": (7,)} + ) + def setup_environment(): parser = setup_common_args_and_variables() diff --git a/backends/qualcomm/tests/test_qnn_delegate_x86.py b/backends/qualcomm/tests/test_qnn_delegate_x86.py index a687f1a334f..7334fb464bf 100644 --- a/backends/qualcomm/tests/test_qnn_delegate_x86.py +++ b/backends/qualcomm/tests/test_qnn_delegate_x86.py @@ -45,3 +45,7 @@ class TestQNNFloatingPointOperator(_ops.TestQNNFloatingPointOperator): class TestQNNQuantizedOperator(_ops.TestQNNQuantizedOperator): pass + + +class TestQNNFcb(_ops.TestQNNFcb): + pass diff --git a/backends/qualcomm/utils/qnn_manager_lifecycle.py b/backends/qualcomm/utils/qnn_manager_lifecycle.py index dbde22f3dbb..3542c25f0fb 100644 --- a/backends/qualcomm/utils/qnn_manager_lifecycle.py +++ b/backends/qualcomm/utils/qnn_manager_lifecycle.py @@ -53,9 +53,19 @@ def get_or_create_qnn_manager( return self._registry[key] def destroy_all(self): - for qnn_manager in self._registry.values(): - qnn_manager.Destroy() - self._registry.clear() + errors = [] + try: + for qnn_manager in self._registry.values(): + try: + qnn_manager.Destroy() + except Exception as error: + errors.append(error) + finally: + self._registry.clear() + if errors: + for error in errors[1:]: + logging.error("Failed to destroy QNN manager: %s", error) + raise errors[0] def _get_target_option( @@ -109,8 +119,10 @@ def QnnManagerContext(compile_specs: Dict[str, List[CompileSpec]]): get_current_qnn_manager(compile_spec_list, soc_model) yield finally: - current_context_registry.destroy_all() - _current_qnn_managers.active_registry = previous_registry + try: + current_context_registry.destroy_all() + finally: + _current_qnn_managers.active_registry = previous_registry def get_current_qnn_manager( diff --git a/backends/qualcomm/utils/utils.py b/backends/qualcomm/utils/utils.py index 1320cd43d37..ce017f2d13b 100644 --- a/backends/qualcomm/utils/utils.py +++ b/backends/qualcomm/utils/utils.py @@ -1276,19 +1276,21 @@ def generate_qnn_executorch_compiler_spec( # noqa: C901 Raises: ValueError: The requested SoC, target pairing, or backend configuration - is unsupported. FCB requires at least two unique HTP SoCs, no online - preparation, and no DLBC when reference-weight sharing is enabled. + is unsupported. FCB requires at least two unique HTP SoCs, an equal + number of SoCs and backend options, no online preparation, and no + DLBC when reference-weight sharing is enabled. """ setup_qnn_sdk() - # Normalize inputs soc_models = soc_model if isinstance(soc_model, list) else [soc_model] target_backend_options = ( backend_options if isinstance(backend_options, list) else [backend_options] ) if len(soc_models) != len(target_backend_options): raise ValueError( - "soc_model and backend_options must have the same number of entries" + "FCB requires equal-length soc_model and backend_options lists" ) + if not soc_models: + raise ValueError("soc_model and backend_options lists must not be empty") is_fcb = len(soc_models) > 1 if len(set(soc_models)) != len(soc_models): raise ValueError("soc_model must contain unique QcomChipset values") @@ -1306,8 +1308,9 @@ def generate_qnn_executorch_compiler_spec( # noqa: C901 if fcb_reference_weight_sharing and any( option.htp_options.use_dlbc for option in target_backend_options ): - warnings.warn( - "FCB reference weight sharing might conflict with DLBC", stacklevel=1 + raise ValueError( + "FCB reference weight sharing does not support DLBC; disable " + "use_dlbc or set fcb_reference_weight_sharing=False" ) htp_arches = {_soc_info_table[model].htp_info.htp_arch for model in soc_models} has_v6x_target = bool(htp_arches & {HtpArch.V68, HtpArch.V69})