diff --git a/backends/test/harness/__init__.py b/backends/test/harness/__init__.py index 3660e979611..d7119afe87a 100644 --- a/backends/test/harness/__init__.py +++ b/backends/test/harness/__init__.py @@ -1,3 +1,4 @@ from .tester import Tester +from .tolerance import get_tolerance, ToleranceConfig -__all__ = ["Tester"] +__all__ = ["Tester", "ToleranceConfig", "get_tolerance"] diff --git a/backends/test/harness/tester.py b/backends/test/harness/tester.py index efe5a14a6c5..f98f3018afe 100644 --- a/backends/test/harness/tester.py +++ b/backends/test/harness/tester.py @@ -44,6 +44,9 @@ def __init__( stage_classes: Dict[StageType, Callable] | None = None, dynamic_shapes: Optional[Tuple[Any]] = None, training: bool = False, + backend: Optional[str] = None, + dtype: torch.dtype = torch.float32, + quantized: bool = False, ): if training: module.train() @@ -54,6 +57,11 @@ def __init__( self.original_module = module self.example_inputs = example_inputs self.dynamic_shapes = dynamic_shapes + self.backend = backend + # Precision profile for tolerance lookup. Explicit — not inferred from + # .quantize() or output dtype (outputs are often dequantized to float). + self.dtype = dtype + self.quantized = quantized self.stages: Dict[StageType, Stage] = OrderedDict.fromkeys(list(StageType)) self.pipeline = { StageType.QUANTIZE: [StageType.EXPORT], @@ -318,13 +326,31 @@ def run_method_and_compare_outputs( stage: Optional[StageType] = None, inputs: Optional[Tuple[torch.Tensor]] = None, num_runs=1, - atol=1e-03, - rtol=1e-03, - qtol=0, + atol=None, + rtol=None, + qtol=None, statistics_callback: Callable[[ErrorStatistics], None] | None = None, artifact_dir: Optional[str] = None, artifact_name: Optional[str] = None, ): + # Resolve tolerances: explicit values win, then registry, then defaults. + if atol is None or rtol is None or qtol is None: + if self.backend is not None: + from executorch.backends.test.harness.tolerance import get_tolerance + + config = get_tolerance( + self.backend, + dtype=self.dtype, + quantized=self.quantized, + ) + atol = atol if atol is not None else config.atol + rtol = rtol if rtol is not None else config.rtol + qtol = qtol if qtol is not None else config.qtol + else: + atol = atol if atol is not None else 1e-03 + rtol = rtol if rtol is not None else 1e-03 + qtol = qtol if qtol is not None else 0 + number_of_runs = 1 if inputs is not None else num_runs reference_stage = self.stages[StageType.EXPORT] diff --git a/backends/test/harness/tests/test_tester.py b/backends/test/harness/tests/test_tester.py new file mode 100644 index 00000000000..b7489d25ea2 --- /dev/null +++ b/backends/test/harness/tests/test_tester.py @@ -0,0 +1,101 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest +from unittest.mock import MagicMock, patch + +import torch +from executorch.backends.test.harness.stages import StageType +from executorch.backends.test.harness.tester import Tester +from executorch.backends.test.harness.tolerance import ToleranceConfig + + +class IdentityModule(torch.nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x + + +class TestTesterToleranceLookup(unittest.TestCase): + def _make_tester( + self, + *, + backend: str, + dtype: torch.dtype = torch.float32, + quantized: bool = False, + ) -> Tester: + module = IdentityModule().eval() + example_inputs = (torch.randn(1, 4, dtype=torch.float32),) + tester = Tester( + module, + example_inputs, + backend=backend, + dtype=dtype, + quantized=quantized, + ) + + # Stub export stage so run_method_and_compare_outputs can run without + # a full ExportedProgram / backend install. + export_stage = MagicMock() + export_stage.artifact = MagicMock() + output = torch.randn(1, 4) + export_stage.run_artifact.return_value = output + tester.stages[StageType.EXPORT] = export_stage + tester.cur = StageType.EXPORT + return tester + + def test_run_method_passes_fp16_dtype_to_get_tolerance(self): + tester = self._make_tester(backend="xnnpack", dtype=torch.float16) + expected = ToleranceConfig(atol=2e-3, rtol=1e-3, qtol=0) + output = torch.randn(1, 4) + + with ( + patch( + "executorch.backends.test.harness.tolerance.get_tolerance", + return_value=expected, + ) as mock_get_tolerance, + patch.object( + Tester, + "_calculate_reference_output", + return_value=(output, None), + ), + patch.object(Tester, "_compare_outputs"), + ): + tester.run_method_and_compare_outputs() + + mock_get_tolerance.assert_called_once_with( + "xnnpack", + dtype=torch.float16, + quantized=False, + ) + + def test_run_method_passes_quantized_flag_to_get_tolerance(self): + tester = self._make_tester(backend="xnnpack", quantized=True) + expected = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1) + output = torch.randn(1, 4) + + with ( + patch( + "executorch.backends.test.harness.tolerance.get_tolerance", + return_value=expected, + ) as mock_get_tolerance, + patch.object( + Tester, + "_calculate_reference_output", + return_value=(output, 0.05), + ), + patch.object(Tester, "_compare_outputs"), + ): + tester.run_method_and_compare_outputs() + + mock_get_tolerance.assert_called_once_with( + "xnnpack", + dtype=torch.float32, + quantized=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/test/harness/tests/test_tolerance.py b/backends/test/harness/tests/test_tolerance.py new file mode 100644 index 00000000000..689679ec8bf --- /dev/null +++ b/backends/test/harness/tests/test_tolerance.py @@ -0,0 +1,245 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest + +import torch +from executorch.backends.test.harness.tolerance import ( + _dtype_to_key, + BACKEND_TOLERANCES, + get_tolerance, + ToleranceConfig, +) + + +class TestToleranceConfig(unittest.TestCase): + def test_default_values(self): + config = ToleranceConfig() + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + self.assertEqual(config.qtol, 0) + + def test_custom_values(self): + config = ToleranceConfig(atol=1e-2, rtol=5e-2, qtol=2) + self.assertEqual(config.atol, 1e-2) + self.assertEqual(config.rtol, 5e-2) + self.assertEqual(config.qtol, 2) + + def test_frozen(self): + config = ToleranceConfig() + with self.assertRaises(AttributeError): + config.atol = 0.5 # type: ignore[misc] + + def test_with_quantization_scale(self): + config = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=2) + scale = 0.05 + adjusted = config.with_quantization_scale(scale) + + self.assertAlmostEqual(adjusted.atol, 1e-3 + 0.05 * 2) + self.assertEqual(adjusted.rtol, 1e-3) + self.assertEqual(adjusted.qtol, 2) + + def test_with_quantization_scale_zero_qtol(self): + config = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + adjusted = config.with_quantization_scale(0.05) + + self.assertEqual(adjusted.atol, 1e-3) + + def test_with_quantization_scale_returns_new_instance(self): + config = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1) + adjusted = config.with_quantization_scale(0.1) + + self.assertIsNot(config, adjusted) + self.assertEqual(config.atol, 1e-3) + + def test_equality(self): + a = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + b = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + self.assertEqual(a, b) + + def test_inequality(self): + a = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + b = ToleranceConfig(atol=1e-2, rtol=1e-3, qtol=0) + self.assertNotEqual(a, b) + + +class TestDtypeToKey(unittest.TestCase): + def test_quantized_always_wins(self): + self.assertEqual(_dtype_to_key(torch.float32, quantized=True), "quantized") + self.assertEqual(_dtype_to_key(torch.float16, quantized=True), "quantized") + self.assertEqual(_dtype_to_key(torch.bfloat16, quantized=True), "quantized") + + def test_fp16(self): + self.assertEqual(_dtype_to_key(torch.float16, quantized=False), "fp16") + + def test_bf16(self): + self.assertEqual(_dtype_to_key(torch.bfloat16, quantized=False), "bf16") + + def test_fp32_default(self): + self.assertEqual(_dtype_to_key(torch.float32, quantized=False), "default") + + def test_fp64_default(self): + self.assertEqual(_dtype_to_key(torch.float64, quantized=False), "default") + + def test_int_types_default(self): + self.assertEqual(_dtype_to_key(torch.int32, quantized=False), "default") + self.assertEqual(_dtype_to_key(torch.int64, quantized=False), "default") + + +class TestGetTolerance(unittest.TestCase): + # --- Known backends --- + + def test_xnnpack_default(self): + config = get_tolerance("xnnpack") + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_xnnpack_fp16(self): + config = get_tolerance("xnnpack", dtype=torch.float16) + self.assertEqual(config.atol, 2e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_xnnpack_quantized(self): + config = get_tolerance("xnnpack", quantized=True) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + self.assertEqual(config.qtol, 1) + + def test_vulkan_default(self): + config = get_tolerance("vulkan") + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-1) + + def test_vulkan_fp16(self): + config = get_tolerance("vulkan", dtype=torch.float16) + self.assertEqual(config.atol, 1e-2) + self.assertEqual(config.rtol, 1e-2) + + def test_coreml_quantized(self): + config = get_tolerance("coreml", quantized=True) + self.assertEqual(config.atol, 5e-2) + self.assertEqual(config.rtol, 5e-2) + + def test_metal_bf16(self): + config = get_tolerance("metal", dtype=torch.bfloat16) + self.assertEqual(config.atol, 1e-2) + self.assertEqual(config.rtol, 1e-2) + + def test_metal_fp32(self): + config = get_tolerance("metal") + self.assertEqual(config.atol, 1e-5) + self.assertEqual(config.rtol, 1e-5) + + def test_qnn_quantized(self): + config = get_tolerance("qnn", quantized=True) + self.assertEqual(config.atol, 1e-1) + self.assertEqual(config.rtol, 1.0) + + def test_arm_quantized(self): + config = get_tolerance("arm", quantized=True) + self.assertEqual(config.qtol, 1) + + def test_arm_bf16(self): + config = get_tolerance("arm", dtype=torch.bfloat16) + self.assertEqual(config.atol, 1e-2) + + def test_nxp_default(self): + config = get_tolerance("nxp") + self.assertEqual(config.atol, 1e-8) + self.assertEqual(config.rtol, 1e-5) + + def test_nxp_quantized(self): + config = get_tolerance("nxp", quantized=True) + self.assertEqual(config.atol, 1.0) + + def test_mlx_fp16(self): + config = get_tolerance("mlx", dtype=torch.float16) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_mlx_bf16(self): + config = get_tolerance("mlx", dtype=torch.bfloat16) + self.assertEqual(config.atol, 1e-2) + self.assertEqual(config.rtol, 1e-2) + + # --- Fallback behavior --- + + def test_unknown_backend_returns_global_default(self): + config = get_tolerance("nonexistent_backend") + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + self.assertEqual(config.qtol, 0) + + def test_unknown_dtype_falls_back_to_backend_default(self): + config = get_tolerance("xnnpack", dtype=torch.float64) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_samsung_no_quantized_falls_back_to_default(self): + config = get_tolerance("samsung", quantized=True) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + def test_webgpu_fp16_falls_back_to_default(self): + config = get_tolerance("webgpu", dtype=torch.float16) + self.assertEqual(config.atol, 1e-3) + self.assertEqual(config.rtol, 1e-3) + + # --- Op parameter (reserved for future use) --- + + def test_op_parameter_currently_ignored(self): + config_with_op = get_tolerance("xnnpack", op="aten.exp") + config_without_op = get_tolerance("xnnpack") + self.assertEqual(config_with_op, config_without_op) + + # --- Registry completeness --- + + def test_all_backends_have_default_key(self): + for backend_name, backend_config in BACKEND_TOLERANCES.items(): + self.assertIn( + "default", + backend_config, + f"Backend '{backend_name}' is missing a 'default' tolerance entry", + ) + + def test_all_tolerance_values_are_non_negative(self): + for backend_name, backend_config in BACKEND_TOLERANCES.items(): + for dtype_key, config in backend_config.items(): + self.assertGreaterEqual( + config.atol, + 0, + f"{backend_name}/{dtype_key} has negative atol", + ) + self.assertGreaterEqual( + config.rtol, + 0, + f"{backend_name}/{dtype_key} has negative rtol", + ) + self.assertGreaterEqual( + config.qtol, + 0, + f"{backend_name}/{dtype_key} has negative qtol", + ) + + def test_registry_has_expected_backends(self): + expected = { + "xnnpack", + "vulkan", + "coreml", + "mps", + "metal", + "qnn", + "arm", + "cortex_m", + "samsung", + "mlx", + "openvino", + "nxp", + "cadence", + "cuda", + "webgpu", + } + self.assertEqual(set(BACKEND_TOLERANCES.keys()), expected) diff --git a/backends/test/harness/tolerance.py b/backends/test/harness/tolerance.py new file mode 100644 index 00000000000..79a53acd9eb --- /dev/null +++ b/backends/test/harness/tolerance.py @@ -0,0 +1,199 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +""" +Centralized tolerance registry for ExecuTorch backend test harness. + +Provides documented per-backend tolerance defaults extracted from existing tests, +a lookup API with fallback chain, and a ToleranceConfig dataclass for consistent +tolerance representation. + +See https://github.com/pytorch/executorch/issues/19910 for context. +""" + +from dataclasses import dataclass +from typing import Dict, Optional + +import torch + + +@dataclass(frozen=True) +class ToleranceConfig: + """Tolerance configuration for numerical accuracy comparison. + + Attributes: + atol: Absolute tolerance for torch.allclose comparison. + rtol: Relative tolerance for torch.allclose comparison. + qtol: Quantization tolerance in quantization steps. When a dequantization + scale is detected, effective atol becomes atol + (scale * qtol). + """ + + atol: float = 1e-3 + rtol: float = 1e-3 + qtol: int = 0 + + def with_quantization_scale(self, scale: float) -> "ToleranceConfig": + """Return a new config with atol adjusted for the given quantization scale.""" + return ToleranceConfig( + atol=self.atol + scale * self.qtol, + rtol=self.rtol, + qtol=self.qtol, + ) + + +# --- Backend tolerance registry --- +# +# Values extracted from a cross-backend audit of all ExecuTorch test files. +# Each backend's defaults reflect what is currently hardcoded across its tests. +# +# Dtype keys: +# "default" — float32 or unspecified dtype +# "fp16" — float16 +# "bf16" — bfloat16 +# "quantized" — int8/uint8 quantized outputs +# +# When adding a new backend, document why the chosen tolerances are appropriate. + +BACKEND_TOLERANCES: Dict[str, Dict[str, ToleranceConfig]] = { + "xnnpack": { + # FP32: 1e-3 is the harness default; ~90% of XNNPACK op tests use this. + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + # FP16: varies per op (exp/gelu use dynamic calc, cos uses 2e-3). + # 2e-3 is a reasonable general default for fp16 ops. + "fp16": ToleranceConfig(atol=2e-3, rtol=1e-3), + # Quantized: most tests use qtol=1; conv2d uses qtol=2. + "quantized": ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1), + }, + "vulkan": { + # Python delegate tests: atol=1e-3, rtol=1e-1. + "default": ToleranceConfig(atol=1e-3, rtol=1e-1), + # FP16: generated op tests and custom ops use 1e-2. + "fp16": ToleranceConfig(atol=1e-2, rtol=1e-2), + # Quantized: delegate tests use atol=1e-2, rtol=1e-1. + "quantized": ToleranceConfig(atol=1e-2, rtol=1e-1), + }, + "coreml": { + # General E2E tests use 1e-2/1e-2. + "default": ToleranceConfig(atol=1e-2, rtol=1e-2), + # Multifunction/KV-cache tests use tighter 1e-4. + "fp16": ToleranceConfig(atol=1e-2, rtol=1e-2), + # Quantizer tests use 5e-2 via np.testing.assert_allclose. + "quantized": ToleranceConfig(atol=5e-2, rtol=5e-2), + }, + "mps": { + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + # FP16 linear tests use atol=5e-2. + "fp16": ToleranceConfig(atol=5e-2, rtol=1e-3), + }, + "metal": { + # FP32 default from DEFAULT_TOLERANCES in test_modules.py. + "default": ToleranceConfig(atol=1e-5, rtol=1e-5), + # BF16 from DEFAULT_TOLERANCES. + "bf16": ToleranceConfig(atol=1e-2, rtol=1e-2), + }, + "qnn": { + # All FP16 HTP/GPU tests: atol=1e-1, rtol=1e-1. + "default": ToleranceConfig(atol=1e-1, rtol=1e-1), + "fp16": ToleranceConfig(atol=1e-1, rtol=1e-1), + # Quantized: atol=1e-1, rtol=1.0. The high rtol reflects + # Qualcomm HTP fixed-point arithmetic variance. + "quantized": ToleranceConfig(atol=1e-1, rtol=1.0), + }, + "arm": { + # Shared harness defaults for TOSA pipeline. + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + "bf16": ToleranceConfig(atol=1e-2, rtol=1e-2), + # INT pipeline default: qtol=1. + "quantized": ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1), + }, + "cortex_m": { + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + # Most quantized op tests use qtol=1; conv uses qtol=2. + "quantized": ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=1), + }, + "samsung": { + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + }, + "mlx": { + # From MLX's TOLERANCE_PRESETS. + "default": ToleranceConfig(atol=1e-5, rtol=1e-5), + "fp16": ToleranceConfig(atol=1e-3, rtol=1e-3), + "bf16": ToleranceConfig(atol=1e-2, rtol=1e-2), + "quantized": ToleranceConfig(atol=1e-1, rtol=1e-1), + }, + "openvino": { + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + }, + "nxp": { + # NXP uses numpy defaults: rtol=1e-5, atol=1e-8. + "default": ToleranceConfig(atol=1e-8, rtol=1e-5), + # Quantized ops typically use atol=1.0 (one int8 step). + "quantized": ToleranceConfig(atol=1.0, rtol=1e-5), + }, + "cadence": { + # Cadence uses RMS-based comparison; these are approximate equivalents + # for the pass test helpers that do use torch.allclose. + "default": ToleranceConfig(atol=1e-6, rtol=1e-5), + "quantized": ToleranceConfig(atol=5.0, rtol=0.05), + }, + "cuda": { + # SDPA tests: MAX_ABS_TOL = 1e-2. + "default": ToleranceConfig(atol=1e-2, rtol=1e-2), + "bf16": ToleranceConfig(atol=1e-2, rtol=1e-2), + }, + "webgpu": { + # Native C++ test: max error threshold 1e-3. + "default": ToleranceConfig(atol=1e-3, rtol=1e-3), + }, +} + +# Global fallback when backend is unknown. +_GLOBAL_DEFAULT = ToleranceConfig(atol=1e-3, rtol=1e-3, qtol=0) + + +def _dtype_to_key(dtype: torch.dtype, quantized: bool) -> str: + """Map a torch dtype and quantization flag to a registry lookup key.""" + if quantized: + return "quantized" + dtype_key_map = { + torch.float16: "fp16", + torch.bfloat16: "bf16", + } + return dtype_key_map.get(dtype, "default") + + +def get_tolerance( + backend: str, + dtype: torch.dtype = torch.float32, + quantized: bool = False, + op: Optional[str] = None, +) -> ToleranceConfig: + """Look up tolerance config for a backend/dtype/op combination. + + Lookup order (first match wins): + 1. Backend + dtype-specific (e.g. xnnpack + fp16) + 2. Backend default + 3. Global default (atol=1e-3, rtol=1e-3, qtol=0) + + Args: + backend: Backend identifier (e.g. "xnnpack", "vulkan", "arm"). + dtype: Output tensor dtype. Used to select fp16/bf16 tolerance tiers. + quantized: Whether the output is from a quantized model. + op: Reserved for future per-op overrides. Currently unused. + + Returns: + ToleranceConfig with appropriate atol, rtol, and qtol values. + """ + backend_config = BACKEND_TOLERANCES.get(backend, {}) + + dtype_key = _dtype_to_key(dtype, quantized) + if dtype_key in backend_config: + return backend_config[dtype_key] + + if "default" in backend_config: + return backend_config["default"] + + return _GLOBAL_DEFAULT