From 43431b91cd83582964faed1031184eb925eb036b Mon Sep 17 00:00:00 2001 From: RiyaP-QA Date: Thu, 4 Jun 2026 12:37:12 +0530 Subject: [PATCH 1/2] Add centralized tolerance registry and integrate with Tester harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a tolerance registry that documents per-backend numerical accuracy defaults extracted from a cross-backend audit of all ExecuTorch test files. Wire it into the Tester harness so backends that identify themselves via the new `backend` parameter get appropriate tolerance defaults automatically. New files: - backends/test/harness/tolerance.py: ToleranceConfig dataclass, BACKEND_TOLERANCES registry (15 backends), get_tolerance() lookup with fallback chain (dtype-specific → backend default → global default) - backends/test/harness/tests/test_tolerance.py: 37 unit tests Tester integration: - New optional `backend` param in Tester.__init__() - run_method_and_compare_outputs() consults registry when tolerances are not explicitly provided - Fully backward-compatible: no backend set = same defaults as before (atol=1e-3, rtol=1e-3, qtol=0); explicit values always win Part of #19910. See #13347 for the broader tolerance strategy discussion. --- backends/test/harness/__init__.py | 3 +- backends/test/harness/tester.py | 22 +- backends/test/harness/tests/test_tolerance.py | 245 ++++++++++++++++++ backends/test/harness/tolerance.py | 199 ++++++++++++++ 4 files changed, 465 insertions(+), 4 deletions(-) create mode 100644 backends/test/harness/tests/test_tolerance.py create mode 100644 backends/test/harness/tolerance.py diff --git a/backends/test/harness/__init__.py b/backends/test/harness/__init__.py index 3660e979611..d7119afe87a 100644 --- a/backends/test/harness/__init__.py +++ b/backends/test/harness/__init__.py @@ -1,3 +1,4 @@ from .tester import Tester +from .tolerance import get_tolerance, ToleranceConfig -__all__ = ["Tester"] +__all__ = ["Tester", "ToleranceConfig", "get_tolerance"] diff --git a/backends/test/harness/tester.py b/backends/test/harness/tester.py index efe5a14a6c5..62f9871c097 100644 --- a/backends/test/harness/tester.py +++ b/backends/test/harness/tester.py @@ -44,6 +44,7 @@ def __init__( stage_classes: Dict[StageType, Callable] | None = None, dynamic_shapes: Optional[Tuple[Any]] = None, training: bool = False, + backend: Optional[str] = None, ): if training: module.train() @@ -54,6 +55,7 @@ def __init__( self.original_module = module self.example_inputs = example_inputs self.dynamic_shapes = dynamic_shapes + self.backend = backend self.stages: Dict[StageType, Stage] = OrderedDict.fromkeys(list(StageType)) self.pipeline = { StageType.QUANTIZE: [StageType.EXPORT], @@ -318,13 +320,27 @@ def run_method_and_compare_outputs( stage: Optional[StageType] = None, inputs: Optional[Tuple[torch.Tensor]] = None, num_runs=1, - atol=1e-03, - rtol=1e-03, - qtol=0, + atol=None, + rtol=None, + qtol=None, statistics_callback: Callable[[ErrorStatistics], None] | None = None, artifact_dir: Optional[str] = None, artifact_name: Optional[str] = None, ): + # Resolve tolerances: explicit values win, then registry, then defaults. + if atol is None or rtol is None or qtol is None: + if self.backend is not None: + from executorch.backends.test.harness.tolerance import get_tolerance + + config = get_tolerance(self.backend) + atol = atol if atol is not None else config.atol + rtol = rtol if rtol is not None else config.rtol + qtol = qtol if qtol is not None else config.qtol + else: + atol = atol if atol is not None else 1e-03 + rtol = rtol if rtol is not None else 1e-03 + qtol = qtol if qtol is not None else 0 + number_of_runs = 1 if inputs is not None else num_runs reference_stage = self.stages[StageType.EXPORT] diff --git a/backends/test/harness/tests/test_tolerance.py b/backends/test/harness/tests/test_tolerance.py new file mode 100644 index 00000000000..689679ec8bf --- /dev/null +++ b/backends/test/harness/tests/test_tolerance.py @@ -0,0 +1,245 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest + +import torch +from executorch.backends.test.harness.tolerance import ( + _dtype_to_key, + BACKEND_TOLERANCES, + get_tolerance, + ToleranceConfig, +) + + +class TestToleranceConfig(unittest.TestCase): + def test_default_values(self): + config = ToleranceConfig() + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + self.assertEqual(config.qtol, 0) + + def test_custom_values(self): + config = ToleranceConfig(atol=1e-2, rtol=5e-2, qtol=2) + self.assertEqual(config.atol, 1e-2) + self.assertEqual(config.rtol, 5e-2) + self.assertEqual(config.qtol, 2) + + def test_frozen(self): + config = ToleranceConfig() + with self.assertRaises(AttributeError): + config.atol = 0.5 # type: ignore[misc] + + def test_with_quantization_scale(self): + config = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=2) + scale = 0.05 + adjusted = config.with_quantization_scale(scale) + + self.assertAlmostEqual(adjusted.atol, 1e-3 + 0.05 * 2) + self.assertEqual(adjusted.rtol, 1e-3) + self.assertEqual(adjusted.qtol, 2) + + def test_with_quantization_scale_zero_qtol(self): + config = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + adjusted = config.with_quantization_scale(0.05) + + self.assertEqual(adjusted.atol, 1e-3) + + def test_with_quantization_scale_returns_new_instance(self): + config = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1) + adjusted = config.with_quantization_scale(0.1) + + self.assertIsNot(config, adjusted) + self.assertEqual(config.atol, 1e-3) + + def test_equality(self): + a = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + b = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + self.assertEqual(a, b) + + def test_inequality(self): + a = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + b = ToleranceConfig(atol=1e-2, rtol=1e-3, qtol=0) + self.assertNotEqual(a, b) + + +class TestDtypeToKey(unittest.TestCase): + def test_quantized_always_wins(self): + self.assertEqual(_dtype_to_key(torch.float32, quantized=True), "quantized") + self.assertEqual(_dtype_to_key(torch.float16, quantized=True), "quantized") + self.assertEqual(_dtype_to_key(torch.bfloat16, quantized=True), "quantized") + + def test_fp16(self): + self.assertEqual(_dtype_to_key(torch.float16, quantized=False), "fp16") + + def test_bf16(self): + self.assertEqual(_dtype_to_key(torch.bfloat16, quantized=False), "bf16") + + def test_fp32_default(self): + self.assertEqual(_dtype_to_key(torch.float32, quantized=False), "default") + + def test_fp64_default(self): + self.assertEqual(_dtype_to_key(torch.float64, quantized=False), "default") + + def test_int_types_default(self): + self.assertEqual(_dtype_to_key(torch.int32, quantized=False), "default") + self.assertEqual(_dtype_to_key(torch.int64, quantized=False), "default") + + +class TestGetTolerance(unittest.TestCase): + # --- Known backends --- + + def test_xnnpack_default(self): + config = get_tolerance("xnnpack") + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_xnnpack_fp16(self): + config = get_tolerance("xnnpack", dtype=torch.float16) + self.assertEqual(config.atol, 2e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_xnnpack_quantized(self): + config = get_tolerance("xnnpack", quantized=True) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + self.assertEqual(config.qtol, 1) + + def test_vulkan_default(self): + config = get_tolerance("vulkan") + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-1) + + def test_vulkan_fp16(self): + config = get_tolerance("vulkan", dtype=torch.float16) + self.assertEqual(config.atol, 1e-2) + self.assertEqual(config.rtol, 1e-2) + + def test_coreml_quantized(self): + config = get_tolerance("coreml", quantized=True) + self.assertEqual(config.atol, 5e-2) + self.assertEqual(config.rtol, 5e-2) + + def test_metal_bf16(self): + config = get_tolerance("metal", dtype=torch.bfloat16) + self.assertEqual(config.atol, 1e-2) + self.assertEqual(config.rtol, 1e-2) + + def test_metal_fp32(self): + config = get_tolerance("metal") + self.assertEqual(config.atol, 1e-5) + self.assertEqual(config.rtol, 1e-5) + + def test_qnn_quantized(self): + config = get_tolerance("qnn", quantized=True) + self.assertEqual(config.atol, 1e-1) + self.assertEqual(config.rtol, 1.0) + + def test_arm_quantized(self): + config = get_tolerance("arm", quantized=True) + self.assertEqual(config.qtol, 1) + + def test_arm_bf16(self): + config = get_tolerance("arm", dtype=torch.bfloat16) + self.assertEqual(config.atol, 1e-2) + + def test_nxp_default(self): + config = get_tolerance("nxp") + self.assertEqual(config.atol, 1e-8) + self.assertEqual(config.rtol, 1e-5) + + def test_nxp_quantized(self): + config = get_tolerance("nxp", quantized=True) + self.assertEqual(config.atol, 1.0) + + def test_mlx_fp16(self): + config = get_tolerance("mlx", dtype=torch.float16) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_mlx_bf16(self): + config = get_tolerance("mlx", dtype=torch.bfloat16) + self.assertEqual(config.atol, 1e-2) + self.assertEqual(config.rtol, 1e-2) + + # --- Fallback behavior --- + + def test_unknown_backend_returns_global_default(self): + config = get_tolerance("nonexistent_backend") + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + self.assertEqual(config.qtol, 0) + + def test_unknown_dtype_falls_back_to_backend_default(self): + config = get_tolerance("xnnpack", dtype=torch.float64) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_samsung_no_quantized_falls_back_to_default(self): + config = get_tolerance("samsung", quantized=True) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_webgpu_fp16_falls_back_to_default(self): + config = get_tolerance("webgpu", dtype=torch.float16) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + # --- Op parameter (reserved for future use) --- + + def test_op_parameter_currently_ignored(self): + config_with_op = get_tolerance("xnnpack", op="aten.exp") + config_without_op = get_tolerance("xnnpack") + self.assertEqual(config_with_op, config_without_op) + + # --- Registry completeness --- + + def test_all_backends_have_default_key(self): + for backend_name, backend_config in BACKEND_TOLERANCES.items(): + self.assertIn( + "default", + backend_config, + f"Backend '{backend_name}' is missing a 'default' tolerance entry", + ) + + def test_all_tolerance_values_are_non_negative(self): + for backend_name, backend_config in BACKEND_TOLERANCES.items(): + for dtype_key, config in backend_config.items(): + self.assertGreaterEqual( + config.atol, + 0, + f"{backend_name}/{dtype_key} has negative atol", + ) + self.assertGreaterEqual( + config.rtol, + 0, + f"{backend_name}/{dtype_key} has negative rtol", + ) + self.assertGreaterEqual( + config.qtol, + 0, + f"{backend_name}/{dtype_key} has negative qtol", + ) + + def test_registry_has_expected_backends(self): + expected = { + "xnnpack", + "vulkan", + "coreml", + "mps", + "metal", + "qnn", + "arm", + "cortex_m", + "samsung", + "mlx", + "openvino", + "nxp", + "cadence", + "cuda", + "webgpu", + } + self.assertEqual(set(BACKEND_TOLERANCES.keys()), expected) diff --git a/backends/test/harness/tolerance.py b/backends/test/harness/tolerance.py new file mode 100644 index 00000000000..79a53acd9eb --- /dev/null +++ b/backends/test/harness/tolerance.py @@ -0,0 +1,199 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +""" +Centralized tolerance registry for ExecuTorch backend test harness. + +Provides documented per-backend tolerance defaults extracted from existing tests, +a lookup API with fallback chain, and a ToleranceConfig dataclass for consistent +tolerance representation. + +See https://github.com/pytorch/executorch/issues/19910 for context. +""" + +from dataclasses import dataclass +from typing import Dict, Optional + +import torch + + +@dataclass(frozen=True) +class ToleranceConfig: + """Tolerance configuration for numerical accuracy comparison. + + Attributes: + atol: Absolute tolerance for torch.allclose comparison. + rtol: Relative tolerance for torch.allclose comparison. + qtol: Quantization tolerance in quantization steps. When a dequantization + scale is detected, effective atol becomes atol + (scale * qtol). + """ + + atol: float = 1e-3 + rtol: float = 1e-3 + qtol: int = 0 + + def with_quantization_scale(self, scale: float) -> "ToleranceConfig": + """Return a new config with atol adjusted for the given quantization scale.""" + return ToleranceConfig( + atol=self.atol + scale * self.qtol, + rtol=self.rtol, + qtol=self.qtol, + ) + + +# --- Backend tolerance registry --- +# +# Values extracted from a cross-backend audit of all ExecuTorch test files. +# Each backend's defaults reflect what is currently hardcoded across its tests. +# +# Dtype keys: +# "default" — float32 or unspecified dtype +# "fp16" — float16 +# "bf16" — bfloat16 +# "quantized" — int8/uint8 quantized outputs +# +# When adding a new backend, document why the chosen tolerances are appropriate. + +BACKEND_TOLERANCES: Dict[str, Dict[str, ToleranceConfig]] = { + "xnnpack": { + # FP32: 1e-3 is the harness default; ~90% of XNNPACK op tests use this. + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + # FP16: varies per op (exp/gelu use dynamic calc, cos uses 2e-3). + # 2e-3 is a reasonable general default for fp16 ops. + "fp16": ToleranceConfig(atol=2e-3, rtol=1e-3), + # Quantized: most tests use qtol=1; conv2d uses qtol=2. + "quantized": ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1), + }, + "vulkan": { + # Python delegate tests: atol=1e-3, rtol=1e-1. + "default": ToleranceConfig(atol=1e-3, rtol=1e-1), + # FP16: generated op tests and custom ops use 1e-2. + "fp16": ToleranceConfig(atol=1e-2, rtol=1e-2), + # Quantized: delegate tests use atol=1e-2, rtol=1e-1. + "quantized": ToleranceConfig(atol=1e-2, rtol=1e-1), + }, + "coreml": { + # General E2E tests use 1e-2/1e-2. + "default": ToleranceConfig(atol=1e-2, rtol=1e-2), + # Multifunction/KV-cache tests use tighter 1e-4. + "fp16": ToleranceConfig(atol=1e-2, rtol=1e-2), + # Quantizer tests use 5e-2 via np.testing.assert_allclose. + "quantized": ToleranceConfig(atol=5e-2, rtol=5e-2), + }, + "mps": { + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + # FP16 linear tests use atol=5e-2. + "fp16": ToleranceConfig(atol=5e-2, rtol=1e-3), + }, + "metal": { + # FP32 default from DEFAULT_TOLERANCES in test_modules.py. + "default": ToleranceConfig(atol=1e-5, rtol=1e-5), + # BF16 from DEFAULT_TOLERANCES. + "bf16": ToleranceConfig(atol=1e-2, rtol=1e-2), + }, + "qnn": { + # All FP16 HTP/GPU tests: atol=1e-1, rtol=1e-1. + "default": ToleranceConfig(atol=1e-1, rtol=1e-1), + "fp16": ToleranceConfig(atol=1e-1, rtol=1e-1), + # Quantized: atol=1e-1, rtol=1.0. The high rtol reflects + # Qualcomm HTP fixed-point arithmetic variance. + "quantized": ToleranceConfig(atol=1e-1, rtol=1.0), + }, + "arm": { + # Shared harness defaults for TOSA pipeline. + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + "bf16": ToleranceConfig(atol=1e-2, rtol=1e-2), + # INT pipeline default: qtol=1. + "quantized": ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1), + }, + "cortex_m": { + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + # Most quantized op tests use qtol=1; conv uses qtol=2. + "quantized": ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1), + }, + "samsung": { + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + }, + "mlx": { + # From MLX's TOLERANCE_PRESETS. + "default": ToleranceConfig(atol=1e-5, rtol=1e-5), + "fp16": ToleranceConfig(atol=1e-3, rtol=1e-3), + "bf16": ToleranceConfig(atol=1e-2, rtol=1e-2), + "quantized": ToleranceConfig(atol=1e-1, rtol=1e-1), + }, + "openvino": { + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + }, + "nxp": { + # NXP uses numpy defaults: rtol=1e-5, atol=1e-8. + "default": ToleranceConfig(atol=1e-8, rtol=1e-5), + # Quantized ops typically use atol=1.0 (one int8 step). + "quantized": ToleranceConfig(atol=1.0, rtol=1e-5), + }, + "cadence": { + # Cadence uses RMS-based comparison; these are approximate equivalents + # for the pass test helpers that do use torch.allclose. + "default": ToleranceConfig(atol=1e-6, rtol=1e-5), + "quantized": ToleranceConfig(atol=5.0, rtol=0.05), + }, + "cuda": { + # SDPA tests: MAX_ABS_TOL = 1e-2. + "default": ToleranceConfig(atol=1e-2, rtol=1e-2), + "bf16": ToleranceConfig(atol=1e-2, rtol=1e-2), + }, + "webgpu": { + # Native C++ test: max error threshold 1e-3. + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + }, +} + +# Global fallback when backend is unknown. +_GLOBAL_DEFAULT = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + + +def _dtype_to_key(dtype: torch.dtype, quantized: bool) -> str: + """Map a torch dtype and quantization flag to a registry lookup key.""" + if quantized: + return "quantized" + dtype_key_map = { + torch.float16: "fp16", + torch.bfloat16: "bf16", + } + return dtype_key_map.get(dtype, "default") + + +def get_tolerance( + backend: str, + dtype: torch.dtype = torch.float32, + quantized: bool = False, + op: Optional[str] = None, +) -> ToleranceConfig: + """Look up tolerance config for a backend/dtype/op combination. + + Lookup order (first match wins): + 1. Backend + dtype-specific (e.g. xnnpack + fp16) + 2. Backend default + 3. Global default (atol=1e-3, rtol=1e-3, qtol=0) + + Args: + backend: Backend identifier (e.g. "xnnpack", "vulkan", "arm"). + dtype: Output tensor dtype. Used to select fp16/bf16 tolerance tiers. + quantized: Whether the output is from a quantized model. + op: Reserved for future per-op overrides. Currently unused. + + Returns: + ToleranceConfig with appropriate atol, rtol, and qtol values. + """ + backend_config = BACKEND_TOLERANCES.get(backend, {}) + + dtype_key = _dtype_to_key(dtype, quantized) + if dtype_key in backend_config: + return backend_config[dtype_key] + + if "default" in backend_config: + return backend_config["default"] + + return _GLOBAL_DEFAULT From 3273215f5fb032a5070778acd251e65bdd13ccd4 Mon Sep 17 00:00:00 2001 From: RiyaP-QA Date: Tue, 3 Nov 2026 01:13:48 +0530 Subject: [PATCH 2/2] Pass dtype and quantized through Tester into tolerance lookup. Avoid inferring precision from output tensors (often dequantized to float); let the test/backend config select the profile, and cover it with harness spy tests. --- backends/test/harness/tester.py | 12 ++- backends/test/harness/tests/test_tester.py | 101 +++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 backends/test/harness/tests/test_tester.py diff --git a/backends/test/harness/tester.py b/backends/test/harness/tester.py index 62f9871c097..f98f3018afe 100644 --- a/backends/test/harness/tester.py +++ b/backends/test/harness/tester.py @@ -45,6 +45,8 @@ def __init__( dynamic_shapes: Optional[Tuple[Any]] = None, training: bool = False, backend: Optional[str] = None, + dtype: torch.dtype = torch.float32, + quantized: bool = False, ): if training: module.train() @@ -56,6 +58,10 @@ def __init__( self.example_inputs = example_inputs self.dynamic_shapes = dynamic_shapes self.backend = backend + # Precision profile for tolerance lookup. Explicit — not inferred from + # .quantize() or output dtype (outputs are often dequantized to float). + self.dtype = dtype + self.quantized = quantized self.stages: Dict[StageType, Stage] = OrderedDict.fromkeys(list(StageType)) self.pipeline = { StageType.QUANTIZE: [StageType.EXPORT], @@ -332,7 +338,11 @@ def run_method_and_compare_outputs( if self.backend is not None: from executorch.backends.test.harness.tolerance import get_tolerance - config = get_tolerance(self.backend) + config = get_tolerance( + self.backend, + dtype=self.dtype, + quantized=self.quantized, + ) atol = atol if atol is not None else config.atol rtol = rtol if rtol is not None else config.rtol qtol = qtol if qtol is not None else config.qtol diff --git a/backends/test/harness/tests/test_tester.py b/backends/test/harness/tests/test_tester.py new file mode 100644 index 00000000000..b7489d25ea2 --- /dev/null +++ b/backends/test/harness/tests/test_tester.py @@ -0,0 +1,101 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest +from unittest.mock import MagicMock, patch + +import torch +from executorch.backends.test.harness.stages import StageType +from executorch.backends.test.harness.tester import Tester +from executorch.backends.test.harness.tolerance import ToleranceConfig + + +class IdentityModule(torch.nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x + + +class TestTesterToleranceLookup(unittest.TestCase): + def _make_tester( + self, + *, + backend: str, + dtype: torch.dtype = torch.float32, + quantized: bool = False, + ) -> Tester: + module = IdentityModule().eval() + example_inputs = (torch.randn(1, 4, dtype=torch.float32),) + tester = Tester( + module, + example_inputs, + backend=backend, + dtype=dtype, + quantized=quantized, + ) + + # Stub export stage so run_method_and_compare_outputs can run without + # a full ExportedProgram / backend install. + export_stage = MagicMock() + export_stage.artifact = MagicMock() + output = torch.randn(1, 4) + export_stage.run_artifact.return_value = output + tester.stages[StageType.EXPORT] = export_stage + tester.cur = StageType.EXPORT + return tester + + def test_run_method_passes_fp16_dtype_to_get_tolerance(self): + tester = self._make_tester(backend="xnnpack", dtype=torch.float16) + expected = ToleranceConfig(atol=2e-3, rtol=1e-3, qtol=0) + output = torch.randn(1, 4) + + with ( + patch( + "executorch.backends.test.harness.tolerance.get_tolerance", + return_value=expected, + ) as mock_get_tolerance, + patch.object( + Tester, + "_calculate_reference_output", + return_value=(output, None), + ), + patch.object(Tester, "_compare_outputs"), + ): + tester.run_method_and_compare_outputs() + + mock_get_tolerance.assert_called_once_with( + "xnnpack", + dtype=torch.float16, + quantized=False, + ) + + def test_run_method_passes_quantized_flag_to_get_tolerance(self): + tester = self._make_tester(backend="xnnpack", quantized=True) + expected = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1) + output = torch.randn(1, 4) + + with ( + patch( + "executorch.backends.test.harness.tolerance.get_tolerance", + return_value=expected, + ) as mock_get_tolerance, + patch.object( + Tester, + "_calculate_reference_output", + return_value=(output, 0.05), + ), + patch.object(Tester, "_compare_outputs"), + ): + tester.run_method_and_compare_outputs() + + mock_get_tolerance.assert_called_once_with( + "xnnpack", + dtype=torch.float32, + quantized=True, + ) + + +if __name__ == "__main__": + unittest.main()