|
6 | 6 |
|
7 | 7 | import unittest |
8 | 8 | from collections import Counter |
| 9 | +from dataclasses import dataclass |
| 10 | +from typing import Optional |
9 | 11 |
|
10 | 12 | import torch |
11 | 13 | import torch.nn as nn |
12 | 14 |
|
| 15 | +from executorch.backends.apple.coreai import ( |
| 16 | + get_default_compile_config, |
| 17 | + get_default_passes, |
| 18 | +) |
13 | 19 | from executorch.backends.apple.coreai.partition.partitioner import CoreAIPartitioner |
14 | 20 | from executorch.backends.apple.coreai.quantizer.quantizer import CoreAIQuantizer |
15 | 21 | from executorch.exir import to_edge_transform_and_lower |
@@ -55,6 +61,69 @@ def _full_finalize(model, example_inputs): |
55 | 61 | return q.finalize(backend=ExportBackend.CoreAI) |
56 | 62 |
|
57 | 63 |
|
| 64 | +def _constant_dtypes(graph_module): |
| 65 | + """dtype of every constant the converted graph holds, keyed by name. |
| 66 | +
|
| 67 | + Targets can be dotted attribute paths (``0.bias``), which a plain getattr |
| 68 | + does not resolve, so walk them: a helper that silently returns nothing |
| 69 | + would make the "no such constant" assertions below pass vacuously. |
| 70 | + """ |
| 71 | + dtypes = {} |
| 72 | + for node in graph_module.graph.nodes: |
| 73 | + if node.op != "get_attr": |
| 74 | + continue |
| 75 | + value = graph_module |
| 76 | + for part in str(node.target).split("."): |
| 77 | + value = getattr(value, part, None) |
| 78 | + if value is None: |
| 79 | + break |
| 80 | + if isinstance(value, torch.Tensor): |
| 81 | + dtypes[str(node.target)] = value.dtype |
| 82 | + return dtypes |
| 83 | + |
| 84 | + |
| 85 | +def _weight_only_config(dtype, scale_dtype=None, granularity=None): |
| 86 | + """Weight-only config: state spec set, input and output specs left off. |
| 87 | +
|
| 88 | + Weights live on the state spec, keyed by name; ``op_input_spec`` and |
| 89 | + ``op_output_spec`` take integer indices and cover activations, so omitting |
| 90 | + them is what makes a config weight-only. |
| 91 | + """ |
| 92 | + from coreai_opt.quantization import ( |
| 93 | + ModuleQuantizerConfig, |
| 94 | + QuantizationSpec, |
| 95 | + QuantizerConfig, |
| 96 | + ) |
| 97 | + |
| 98 | + spec = QuantizationSpec( |
| 99 | + dtype=dtype, |
| 100 | + scale_dtype=scale_dtype, |
| 101 | + granularity=granularity or {"type": "per_channel", "axis": 0}, |
| 102 | + ) |
| 103 | + return QuantizerConfig( |
| 104 | + global_config=ModuleQuantizerConfig( |
| 105 | + op_state_spec={"weight": spec}, |
| 106 | + op_input_spec=None, |
| 107 | + op_output_spec=None, |
| 108 | + ) |
| 109 | + ) |
| 110 | + |
| 111 | + |
| 112 | +@dataclass(frozen=True) |
| 113 | +class FloatQuantCase: |
| 114 | + """One FP weight-quantization configuration and what it should produce.""" |
| 115 | + |
| 116 | + name: str |
| 117 | + dtype: torch.dtype |
| 118 | + scale_dtype: Optional[torch.dtype] |
| 119 | + granularity: dict |
| 120 | + weight_dtype: torch.dtype |
| 121 | + scale_out_dtype: torch.dtype |
| 122 | + |
| 123 | + def config(self): |
| 124 | + return _weight_only_config(self.dtype, self.scale_dtype, self.granularity) |
| 125 | + |
| 126 | + |
58 | 127 | class CoreAIQuantizerTest(unittest.TestCase): |
59 | 128 | def setUp(self): |
60 | 129 | self.example_inputs = (torch.randn(2, 32),) |
@@ -187,5 +256,257 @@ def test_quantized_lowers_through_to_executorch(self): |
187 | 256 | self.assertGreater(len(buffer), 0) |
188 | 257 |
|
189 | 258 |
|
| 259 | +@dataclass(frozen=True) |
| 260 | +class IntWeightOnlyCase: |
| 261 | + """One integer weight-only configuration. |
| 262 | +
|
| 263 | + ``preset`` names a ``QuantizerConfig.presets`` constructor where Apple |
| 264 | + ships one. int2 has none, so its specs are built to match the shape the |
| 265 | + presets use: signed, symmetric. |
| 266 | + """ |
| 267 | + |
| 268 | + name: str |
| 269 | + preset: Optional[str] |
| 270 | + dtype: Optional[torch.dtype] = None |
| 271 | + granularity: Optional[dict] = None |
| 272 | + |
| 273 | + def config(self): |
| 274 | + if self.preset is not None: |
| 275 | + from coreai_opt.quantization import QuantizerConfig |
| 276 | + |
| 277 | + return getattr(QuantizerConfig.presets, self.preset)() |
| 278 | + return _weight_only_config(self.dtype, granularity=self.granularity) |
| 279 | + |
| 280 | + |
| 281 | +class IntegerWeightOnlyQuantizationTest(unittest.TestCase): |
| 282 | + """Signed integer weight-only quantization across widths. |
| 283 | +
|
| 284 | + Signed symmetric is the sanctioned sub-byte path: every preset Apple ships |
| 285 | + (``w8``, ``w4``, ``w4_per_block``) is signed and symmetric, and there is no |
| 286 | + unsigned or asymmetric preset. The unsigned and affine variants do lower, |
| 287 | + but they are not what the presets select. |
| 288 | +
|
| 289 | + int2 is covered here because it lowers cleanly, but note there is no int2 |
| 290 | + preset at all, in either granularity. |
| 291 | +
|
| 292 | + The default config quantizes activations as well, so these cover a |
| 293 | + configuration the tests above do not reach. |
| 294 | + """ |
| 295 | + |
| 296 | + CASES = ( |
| 297 | + IntWeightOnlyCase(name="int8", preset="w8"), |
| 298 | + IntWeightOnlyCase(name="int4", preset="w4"), |
| 299 | + IntWeightOnlyCase(name="int4_per_block", preset="w4_per_block"), |
| 300 | + IntWeightOnlyCase(name="int2", preset=None, dtype=torch.int2), |
| 301 | + IntWeightOnlyCase( |
| 302 | + # Per-block is the more sensible choice at 2 bits: more scales to |
| 303 | + # compensate for a four-level grid. |
| 304 | + name="int2_per_block", |
| 305 | + preset=None, |
| 306 | + dtype=torch.int2, |
| 307 | + granularity={"type": "per_block", "block_size": (1, 16)}, |
| 308 | + ), |
| 309 | + ) |
| 310 | + |
| 311 | + def setUp(self): |
| 312 | + self.example_inputs = (torch.randn(2, 32),) |
| 313 | + |
| 314 | + def _convert(self, config): |
| 315 | + quantizer = CoreAIQuantizer(_model(), config) |
| 316 | + prepared = quantizer.prepare(self.example_inputs) |
| 317 | + with quantizer.calibration_mode(): |
| 318 | + prepared(*self.example_inputs) |
| 319 | + return quantizer.convert() |
| 320 | + |
| 321 | + def _pte_size(self, config): |
| 322 | + ep = torch.export.export(self._convert(config), self.example_inputs) |
| 323 | + lowered = to_edge_transform_and_lower( |
| 324 | + ep, |
| 325 | + transform_passes=get_default_passes(), |
| 326 | + partitioner=[CoreAIPartitioner()], |
| 327 | + compile_config=get_default_compile_config(), |
| 328 | + ) |
| 329 | + return len(bytes(lowered.to_executorch().buffer)) |
| 330 | + |
| 331 | + def test_activations_are_left_unquantized(self): |
| 332 | + """The property that makes these weight-only, rather than just narrow.""" |
| 333 | + for case in self.CASES: |
| 334 | + with self.subTest(case.name): |
| 335 | + converted = self._convert(case.config()) |
| 336 | + ops = Counter(_coreai_quant_op_names(converted)) |
| 337 | + self.assertEqual(ops["quantize"], 0) |
| 338 | + self.assertEqual(ops["dequantize"], 0) |
| 339 | + self.assertGreater(ops["constexpr_blockwise_shift_scale"], 0) |
| 340 | + |
| 341 | + constants = _constant_dtypes(converted) |
| 342 | + self.assertEqual( |
| 343 | + [n for n in constants if n.startswith("activation_post_process")], |
| 344 | + [], |
| 345 | + ) |
| 346 | + |
| 347 | + def test_default_config_quantizes_activations(self): |
| 348 | + """Contrast: the default config does insert activation quant. |
| 349 | +
|
| 350 | + Without this the weight-only assertions could pass against a build |
| 351 | + that had stopped quantizing activations for everyone. |
| 352 | + """ |
| 353 | + ops = Counter(_coreai_quant_op_names(self._convert(None))) |
| 354 | + self.assertGreater(ops["quantize"], 0) |
| 355 | + self.assertGreater(ops["dequantize"], 0) |
| 356 | + |
| 357 | + def test_each_width_lowers_through_to_executorch(self): |
| 358 | + for case in self.CASES: |
| 359 | + with self.subTest(case.name): |
| 360 | + self.assertGreater(self._pte_size(case.config()), 0) |
| 361 | + |
| 362 | + def test_smaller_blocks_add_scales(self): |
| 363 | + """Per-block at 2 bits trades size for accuracy, so it must cost more. |
| 364 | +
|
| 365 | + Confirms the granularity is applied rather than silently ignored: the |
| 366 | + block-16 variant carries more scale constants than per-channel. |
| 367 | + """ |
| 368 | + by_name = {case.name: case for case in self.CASES} |
| 369 | + per_channel = self._pte_size(by_name["int2"].config()) |
| 370 | + per_block = self._pte_size(by_name["int2_per_block"].config()) |
| 371 | + self.assertGreater( |
| 372 | + per_block, |
| 373 | + per_channel, |
| 374 | + f"block-16 int2 ({per_block}) should exceed per-channel " |
| 375 | + f"({per_channel}); equal sizes suggest granularity was ignored", |
| 376 | + ) |
| 377 | + |
| 378 | + def test_narrower_weights_produce_smaller_programs(self): |
| 379 | + """Guards a silent fallback that would keep the wider dtype. |
| 380 | +
|
| 381 | + Sub-byte weights surface as int8 in the fx graph (the byte container, |
| 382 | + packed later), so the asset size is the honest observable rather than |
| 383 | + the constant's dtype. |
| 384 | + """ |
| 385 | + sizes = { |
| 386 | + case.name: self._pte_size(case.config()) |
| 387 | + for case in self.CASES |
| 388 | + if case.name in ("int2", "int4", "int8") |
| 389 | + } |
| 390 | + self.assertLess(sizes["int2"], sizes["int4"], sizes) |
| 391 | + self.assertLess(sizes["int4"], sizes["int8"], sizes) |
| 392 | + |
| 393 | + |
| 394 | +class FloatQuantizationTest(unittest.TestCase): |
| 395 | + """FP8 and FP4 weight quantization, end to end. |
| 396 | +
|
| 397 | + coreai_opt expresses these as ordinary ``torch.dtype`` values on a |
| 398 | + ``QuantizationSpec``, with ``scale_dtype`` selecting the scale form: an |
| 399 | + fp32 scale for plain FP8, or a power-of-two ``float8_e8m0fnu`` scale for |
| 400 | + OCP Microscaling (FP4 always, FP8 optionally). |
| 401 | + """ |
| 402 | + |
| 403 | + CASES = ( |
| 404 | + FloatQuantCase( |
| 405 | + name="fp8_e4m3_per_channel", |
| 406 | + dtype=torch.float8_e4m3fn, |
| 407 | + scale_dtype=None, |
| 408 | + granularity={"type": "per_channel", "axis": 0}, |
| 409 | + weight_dtype=torch.float8_e4m3fn, |
| 410 | + scale_out_dtype=torch.float32, |
| 411 | + ), |
| 412 | + FloatQuantCase( |
| 413 | + name="fp8_e5m2_per_channel", |
| 414 | + dtype=torch.float8_e5m2, |
| 415 | + scale_dtype=None, |
| 416 | + granularity={"type": "per_channel", "axis": 0}, |
| 417 | + weight_dtype=torch.float8_e5m2, |
| 418 | + scale_out_dtype=torch.float32, |
| 419 | + ), |
| 420 | + FloatQuantCase( |
| 421 | + name="fp8_e4m3_microscaling", |
| 422 | + dtype=torch.float8_e4m3fn, |
| 423 | + scale_dtype=torch.float8_e8m0fnu, |
| 424 | + granularity={"type": "per_block", "block_size": (1, 32)}, |
| 425 | + weight_dtype=torch.float8_e4m3fn, |
| 426 | + scale_out_dtype=torch.float8_e8m0fnu, |
| 427 | + ), |
| 428 | + FloatQuantCase( |
| 429 | + # FP4 is packed two values per byte, so the quantized weight |
| 430 | + # surfaces as uint8; coreai-torch recovers the logical shape. |
| 431 | + name="fp4_e2m1_microscaling", |
| 432 | + dtype=torch.float4_e2m1fn_x2, |
| 433 | + scale_dtype=None, |
| 434 | + granularity={"type": "per_block", "block_size": (1, 32)}, |
| 435 | + weight_dtype=torch.uint8, |
| 436 | + scale_out_dtype=torch.float8_e8m0fnu, |
| 437 | + ), |
| 438 | + ) |
| 439 | + |
| 440 | + def setUp(self): |
| 441 | + self.example_inputs = (torch.randn(2, 32),) |
| 442 | + |
| 443 | + def _convert(self, case): |
| 444 | + quantizer = CoreAIQuantizer(_model(), case.config()) |
| 445 | + prepared = quantizer.prepare(self.example_inputs) |
| 446 | + with quantizer.calibration_mode(): |
| 447 | + prepared(*self.example_inputs) |
| 448 | + return quantizer.convert() |
| 449 | + |
| 450 | + def _pte_size(self, case): |
| 451 | + converted = self._convert(case) |
| 452 | + ep = torch.export.export(converted, self.example_inputs) |
| 453 | + lowered = to_edge_transform_and_lower( |
| 454 | + ep, |
| 455 | + transform_passes=get_default_passes(), |
| 456 | + partitioner=[CoreAIPartitioner()], |
| 457 | + compile_config=get_default_compile_config(), |
| 458 | + ) |
| 459 | + return len(bytes(lowered.to_executorch().buffer)) |
| 460 | + |
| 461 | + def test_quantized_weight_and_scale_dtypes(self): |
| 462 | + """The requested dtype must reach the graph, not silently degrade.""" |
| 463 | + for case in self.CASES: |
| 464 | + with self.subTest(case.name): |
| 465 | + constants = _constant_dtypes(self._convert(case)) |
| 466 | + weights = [ |
| 467 | + dtype |
| 468 | + for name, dtype in constants.items() |
| 469 | + if name.endswith("weight_quantized") |
| 470 | + ] |
| 471 | + scales = [ |
| 472 | + dtype |
| 473 | + for name, dtype in constants.items() |
| 474 | + if name.endswith("weight_scale") |
| 475 | + ] |
| 476 | + self.assertTrue(weights, f"no quantized weight in {constants}") |
| 477 | + self.assertEqual(set(weights), {case.weight_dtype}) |
| 478 | + self.assertEqual(set(scales), {case.scale_out_dtype}) |
| 479 | + |
| 480 | + def test_float_quantization_has_no_zero_point(self): |
| 481 | + """FP formats are symmetric only, so zero-point is always implicit.""" |
| 482 | + for case in self.CASES: |
| 483 | + with self.subTest(case.name): |
| 484 | + constants = _constant_dtypes(self._convert(case)) |
| 485 | + self.assertTrue(constants, "no constants collected") |
| 486 | + self.assertEqual([n for n in constants if n.endswith("zero_point")], []) |
| 487 | + |
| 488 | + def test_lowers_through_to_executorch(self): |
| 489 | + for case in self.CASES: |
| 490 | + with self.subTest(case.name): |
| 491 | + self.assertGreater(self._pte_size(case), 0) |
| 492 | + |
| 493 | + def test_fp4_is_smaller_than_fp8(self): |
| 494 | + """Guards against a silent fallback that would keep the wider dtype. |
| 495 | +
|
| 496 | + Both sides use the same per-block granularity and e8m0 scales, so the |
| 497 | + difference is the weight dtype alone. |
| 498 | + """ |
| 499 | + by_name = {case.name: case for case in self.CASES} |
| 500 | + fp4 = self._pte_size(by_name["fp4_e2m1_microscaling"]) |
| 501 | + fp8 = self._pte_size(by_name["fp8_e4m3_microscaling"]) |
| 502 | + self.assertLess( |
| 503 | + fp4, |
| 504 | + fp8, |
| 505 | + f"fp4 pte ({fp4}) should be smaller than fp8 at the same " |
| 506 | + f"granularity ({fp8}); equal sizes suggest the requested dtype was " |
| 507 | + f"not applied", |
| 508 | + ) |
| 509 | + |
| 510 | + |
190 | 511 | if __name__ == "__main__": |
191 | 512 | unittest.main() |
0 commit comments