Skip to content

Commit 3447ba9

Browse files
authored
1 parent b0a80bc commit 3447ba9

2 files changed

Lines changed: 328 additions & 1 deletion

File tree

‎backends/apple/coreai/__init__.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,14 @@ def get_default_compile_config() -> EdgeCompileConfig:
3434
``_skip_dim_order=True`` keeps ExecuTorch on ``aten._to_copy`` instead of
3535
emitting ``dim_order_ops._to_dim_order_copy``, which coreai-torch cannot
3636
lower (its validator requires dim-order ops be decomposed).
37+
38+
``_check_ir_validity=False`` disables the edge-dialect verifier. Core AI's
39+
quantized ops legitimately mix dtypes, which the verifier forbids: an FP8 or
40+
FP4 weight reaches ``coreai::constexpr_blockwise_shift_scale`` alongside an
41+
fp32 or ``float8_e8m0fnu`` scale, and the check rejects any op whose tensor
42+
inputs disagree on dtype.
3743
"""
38-
return EdgeCompileConfig(_skip_dim_order=True)
44+
return EdgeCompileConfig(_skip_dim_order=True, _check_ir_validity=False)
3945

4046

4147
__all__ = [

‎backends/apple/coreai/test/test_quantizer.py‎

Lines changed: 321 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,16 @@
66

77
import unittest
88
from collections import Counter
9+
from dataclasses import dataclass
10+
from typing import Optional
911

1012
import torch
1113
import torch.nn as nn
1214

15+
from executorch.backends.apple.coreai import (
16+
get_default_compile_config,
17+
get_default_passes,
18+
)
1319
from executorch.backends.apple.coreai.partition.partitioner import CoreAIPartitioner
1420
from executorch.backends.apple.coreai.quantizer.quantizer import CoreAIQuantizer
1521
from executorch.exir import to_edge_transform_and_lower
@@ -55,6 +61,69 @@ def _full_finalize(model, example_inputs):
5561
return q.finalize(backend=ExportBackend.CoreAI)
5662

5763

64+
def _constant_dtypes(graph_module):
65+
"""dtype of every constant the converted graph holds, keyed by name.
66+
67+
Targets can be dotted attribute paths (``0.bias``), which a plain getattr
68+
does not resolve, so walk them: a helper that silently returns nothing
69+
would make the "no such constant" assertions below pass vacuously.
70+
"""
71+
dtypes = {}
72+
for node in graph_module.graph.nodes:
73+
if node.op != "get_attr":
74+
continue
75+
value = graph_module
76+
for part in str(node.target).split("."):
77+
value = getattr(value, part, None)
78+
if value is None:
79+
break
80+
if isinstance(value, torch.Tensor):
81+
dtypes[str(node.target)] = value.dtype
82+
return dtypes
83+
84+
85+
def _weight_only_config(dtype, scale_dtype=None, granularity=None):
86+
"""Weight-only config: state spec set, input and output specs left off.
87+
88+
Weights live on the state spec, keyed by name; ``op_input_spec`` and
89+
``op_output_spec`` take integer indices and cover activations, so omitting
90+
them is what makes a config weight-only.
91+
"""
92+
from coreai_opt.quantization import (
93+
ModuleQuantizerConfig,
94+
QuantizationSpec,
95+
QuantizerConfig,
96+
)
97+
98+
spec = QuantizationSpec(
99+
dtype=dtype,
100+
scale_dtype=scale_dtype,
101+
granularity=granularity or {"type": "per_channel", "axis": 0},
102+
)
103+
return QuantizerConfig(
104+
global_config=ModuleQuantizerConfig(
105+
op_state_spec={"weight": spec},
106+
op_input_spec=None,
107+
op_output_spec=None,
108+
)
109+
)
110+
111+
112+
@dataclass(frozen=True)
113+
class FloatQuantCase:
114+
"""One FP weight-quantization configuration and what it should produce."""
115+
116+
name: str
117+
dtype: torch.dtype
118+
scale_dtype: Optional[torch.dtype]
119+
granularity: dict
120+
weight_dtype: torch.dtype
121+
scale_out_dtype: torch.dtype
122+
123+
def config(self):
124+
return _weight_only_config(self.dtype, self.scale_dtype, self.granularity)
125+
126+
58127
class CoreAIQuantizerTest(unittest.TestCase):
59128
def setUp(self):
60129
self.example_inputs = (torch.randn(2, 32),)
@@ -187,5 +256,257 @@ def test_quantized_lowers_through_to_executorch(self):
187256
self.assertGreater(len(buffer), 0)
188257

189258

259+
@dataclass(frozen=True)
260+
class IntWeightOnlyCase:
261+
"""One integer weight-only configuration.
262+
263+
``preset`` names a ``QuantizerConfig.presets`` constructor where Apple
264+
ships one. int2 has none, so its specs are built to match the shape the
265+
presets use: signed, symmetric.
266+
"""
267+
268+
name: str
269+
preset: Optional[str]
270+
dtype: Optional[torch.dtype] = None
271+
granularity: Optional[dict] = None
272+
273+
def config(self):
274+
if self.preset is not None:
275+
from coreai_opt.quantization import QuantizerConfig
276+
277+
return getattr(QuantizerConfig.presets, self.preset)()
278+
return _weight_only_config(self.dtype, granularity=self.granularity)
279+
280+
281+
class IntegerWeightOnlyQuantizationTest(unittest.TestCase):
282+
"""Signed integer weight-only quantization across widths.
283+
284+
Signed symmetric is the sanctioned sub-byte path: every preset Apple ships
285+
(``w8``, ``w4``, ``w4_per_block``) is signed and symmetric, and there is no
286+
unsigned or asymmetric preset. The unsigned and affine variants do lower,
287+
but they are not what the presets select.
288+
289+
int2 is covered here because it lowers cleanly, but note there is no int2
290+
preset at all, in either granularity.
291+
292+
The default config quantizes activations as well, so these cover a
293+
configuration the tests above do not reach.
294+
"""
295+
296+
CASES = (
297+
IntWeightOnlyCase(name="int8", preset="w8"),
298+
IntWeightOnlyCase(name="int4", preset="w4"),
299+
IntWeightOnlyCase(name="int4_per_block", preset="w4_per_block"),
300+
IntWeightOnlyCase(name="int2", preset=None, dtype=torch.int2),
301+
IntWeightOnlyCase(
302+
# Per-block is the more sensible choice at 2 bits: more scales to
303+
# compensate for a four-level grid.
304+
name="int2_per_block",
305+
preset=None,
306+
dtype=torch.int2,
307+
granularity={"type": "per_block", "block_size": (1, 16)},
308+
),
309+
)
310+
311+
def setUp(self):
312+
self.example_inputs = (torch.randn(2, 32),)
313+
314+
def _convert(self, config):
315+
quantizer = CoreAIQuantizer(_model(), config)
316+
prepared = quantizer.prepare(self.example_inputs)
317+
with quantizer.calibration_mode():
318+
prepared(*self.example_inputs)
319+
return quantizer.convert()
320+
321+
def _pte_size(self, config):
322+
ep = torch.export.export(self._convert(config), self.example_inputs)
323+
lowered = to_edge_transform_and_lower(
324+
ep,
325+
transform_passes=get_default_passes(),
326+
partitioner=[CoreAIPartitioner()],
327+
compile_config=get_default_compile_config(),
328+
)
329+
return len(bytes(lowered.to_executorch().buffer))
330+
331+
def test_activations_are_left_unquantized(self):
332+
"""The property that makes these weight-only, rather than just narrow."""
333+
for case in self.CASES:
334+
with self.subTest(case.name):
335+
converted = self._convert(case.config())
336+
ops = Counter(_coreai_quant_op_names(converted))
337+
self.assertEqual(ops["quantize"], 0)
338+
self.assertEqual(ops["dequantize"], 0)
339+
self.assertGreater(ops["constexpr_blockwise_shift_scale"], 0)
340+
341+
constants = _constant_dtypes(converted)
342+
self.assertEqual(
343+
[n for n in constants if n.startswith("activation_post_process")],
344+
[],
345+
)
346+
347+
def test_default_config_quantizes_activations(self):
348+
"""Contrast: the default config does insert activation quant.
349+
350+
Without this the weight-only assertions could pass against a build
351+
that had stopped quantizing activations for everyone.
352+
"""
353+
ops = Counter(_coreai_quant_op_names(self._convert(None)))
354+
self.assertGreater(ops["quantize"], 0)
355+
self.assertGreater(ops["dequantize"], 0)
356+
357+
def test_each_width_lowers_through_to_executorch(self):
358+
for case in self.CASES:
359+
with self.subTest(case.name):
360+
self.assertGreater(self._pte_size(case.config()), 0)
361+
362+
def test_smaller_blocks_add_scales(self):
363+
"""Per-block at 2 bits trades size for accuracy, so it must cost more.
364+
365+
Confirms the granularity is applied rather than silently ignored: the
366+
block-16 variant carries more scale constants than per-channel.
367+
"""
368+
by_name = {case.name: case for case in self.CASES}
369+
per_channel = self._pte_size(by_name["int2"].config())
370+
per_block = self._pte_size(by_name["int2_per_block"].config())
371+
self.assertGreater(
372+
per_block,
373+
per_channel,
374+
f"block-16 int2 ({per_block}) should exceed per-channel "
375+
f"({per_channel}); equal sizes suggest granularity was ignored",
376+
)
377+
378+
def test_narrower_weights_produce_smaller_programs(self):
379+
"""Guards a silent fallback that would keep the wider dtype.
380+
381+
Sub-byte weights surface as int8 in the fx graph (the byte container,
382+
packed later), so the asset size is the honest observable rather than
383+
the constant's dtype.
384+
"""
385+
sizes = {
386+
case.name: self._pte_size(case.config())
387+
for case in self.CASES
388+
if case.name in ("int2", "int4", "int8")
389+
}
390+
self.assertLess(sizes["int2"], sizes["int4"], sizes)
391+
self.assertLess(sizes["int4"], sizes["int8"], sizes)
392+
393+
394+
class FloatQuantizationTest(unittest.TestCase):
395+
"""FP8 and FP4 weight quantization, end to end.
396+
397+
coreai_opt expresses these as ordinary ``torch.dtype`` values on a
398+
``QuantizationSpec``, with ``scale_dtype`` selecting the scale form: an
399+
fp32 scale for plain FP8, or a power-of-two ``float8_e8m0fnu`` scale for
400+
OCP Microscaling (FP4 always, FP8 optionally).
401+
"""
402+
403+
CASES = (
404+
FloatQuantCase(
405+
name="fp8_e4m3_per_channel",
406+
dtype=torch.float8_e4m3fn,
407+
scale_dtype=None,
408+
granularity={"type": "per_channel", "axis": 0},
409+
weight_dtype=torch.float8_e4m3fn,
410+
scale_out_dtype=torch.float32,
411+
),
412+
FloatQuantCase(
413+
name="fp8_e5m2_per_channel",
414+
dtype=torch.float8_e5m2,
415+
scale_dtype=None,
416+
granularity={"type": "per_channel", "axis": 0},
417+
weight_dtype=torch.float8_e5m2,
418+
scale_out_dtype=torch.float32,
419+
),
420+
FloatQuantCase(
421+
name="fp8_e4m3_microscaling",
422+
dtype=torch.float8_e4m3fn,
423+
scale_dtype=torch.float8_e8m0fnu,
424+
granularity={"type": "per_block", "block_size": (1, 32)},
425+
weight_dtype=torch.float8_e4m3fn,
426+
scale_out_dtype=torch.float8_e8m0fnu,
427+
),
428+
FloatQuantCase(
429+
# FP4 is packed two values per byte, so the quantized weight
430+
# surfaces as uint8; coreai-torch recovers the logical shape.
431+
name="fp4_e2m1_microscaling",
432+
dtype=torch.float4_e2m1fn_x2,
433+
scale_dtype=None,
434+
granularity={"type": "per_block", "block_size": (1, 32)},
435+
weight_dtype=torch.uint8,
436+
scale_out_dtype=torch.float8_e8m0fnu,
437+
),
438+
)
439+
440+
def setUp(self):
441+
self.example_inputs = (torch.randn(2, 32),)
442+
443+
def _convert(self, case):
444+
quantizer = CoreAIQuantizer(_model(), case.config())
445+
prepared = quantizer.prepare(self.example_inputs)
446+
with quantizer.calibration_mode():
447+
prepared(*self.example_inputs)
448+
return quantizer.convert()
449+
450+
def _pte_size(self, case):
451+
converted = self._convert(case)
452+
ep = torch.export.export(converted, self.example_inputs)
453+
lowered = to_edge_transform_and_lower(
454+
ep,
455+
transform_passes=get_default_passes(),
456+
partitioner=[CoreAIPartitioner()],
457+
compile_config=get_default_compile_config(),
458+
)
459+
return len(bytes(lowered.to_executorch().buffer))
460+
461+
def test_quantized_weight_and_scale_dtypes(self):
462+
"""The requested dtype must reach the graph, not silently degrade."""
463+
for case in self.CASES:
464+
with self.subTest(case.name):
465+
constants = _constant_dtypes(self._convert(case))
466+
weights = [
467+
dtype
468+
for name, dtype in constants.items()
469+
if name.endswith("weight_quantized")
470+
]
471+
scales = [
472+
dtype
473+
for name, dtype in constants.items()
474+
if name.endswith("weight_scale")
475+
]
476+
self.assertTrue(weights, f"no quantized weight in {constants}")
477+
self.assertEqual(set(weights), {case.weight_dtype})
478+
self.assertEqual(set(scales), {case.scale_out_dtype})
479+
480+
def test_float_quantization_has_no_zero_point(self):
481+
"""FP formats are symmetric only, so zero-point is always implicit."""
482+
for case in self.CASES:
483+
with self.subTest(case.name):
484+
constants = _constant_dtypes(self._convert(case))
485+
self.assertTrue(constants, "no constants collected")
486+
self.assertEqual([n for n in constants if n.endswith("zero_point")], [])
487+
488+
def test_lowers_through_to_executorch(self):
489+
for case in self.CASES:
490+
with self.subTest(case.name):
491+
self.assertGreater(self._pte_size(case), 0)
492+
493+
def test_fp4_is_smaller_than_fp8(self):
494+
"""Guards against a silent fallback that would keep the wider dtype.
495+
496+
Both sides use the same per-block granularity and e8m0 scales, so the
497+
difference is the weight dtype alone.
498+
"""
499+
by_name = {case.name: case for case in self.CASES}
500+
fp4 = self._pte_size(by_name["fp4_e2m1_microscaling"])
501+
fp8 = self._pte_size(by_name["fp8_e4m3_microscaling"])
502+
self.assertLess(
503+
fp4,
504+
fp8,
505+
f"fp4 pte ({fp4}) should be smaller than fp8 at the same "
506+
f"granularity ({fp8}); equal sizes suggest the requested dtype was "
507+
f"not applied",
508+
)
509+
510+
190511
if __name__ == "__main__":
191512
unittest.main()

0 commit comments

Comments
 (0)