Skip to content

Commit b0a80bc

Browse files
authored
1 parent 34608e3 commit b0a80bc

3 files changed

Lines changed: 353 additions & 0 deletions

File tree

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
from executorch.backends.apple.coreai.palettizer.palettizer import CoreAIPalettizer
8+
9+
__all__ = [
10+
"CoreAIPalettizer",
11+
]
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Core AI palettization front-end for ExecuTorch.
8+
9+
Palettization replaces a weight with a small learned lookup table plus an index
10+
per element, rather than a scale and zero-point. It sits alongside
11+
quantization::
12+
13+
prepare -> (optional sensitivity calibration) -> finalize -> export
14+
15+
Unlike :class:`~executorch.backends.apple.coreai.quantizer.quantizer.CoreAIQuantizer`,
16+
:meth:`finalize` returns an eager ``nn.Module`` whose weights carry a
17+
parametrization, not an fx graph. The compression becomes a ``lut_to_dense`` op
18+
only once the model is exported, so nothing here needs to run before
19+
``torch.export`` for the backend's sake; the ordinary
20+
``to_edge_transform_and_lower`` path lowers it.
21+
22+
Palettization is weight-only by construction, so there is no activation
23+
counterpart to configure.
24+
25+
``coreai_opt`` (the ``coreai-optimization`` package) is imported lazily so this
26+
module stays importable in environments without it installed.
27+
"""
28+
29+
from __future__ import annotations
30+
31+
from typing import Any, Callable, Optional, Sequence
32+
33+
from torch import nn
34+
35+
36+
class CoreAIPalettizer:
37+
"""K-means palettization front-end for Core AI, wrapping ``coreai_opt``.
38+
39+
Args:
40+
model: the ``nn.Module`` to palettize.
41+
config: a ``coreai_opt.palettization.KMeansPalettizerConfig``. Defaults
42+
to ``coreai_opt``'s own default when ``None``. Ready-made
43+
configurations are available as
44+
``KMeansPalettizerConfig.presets.w4()``, ``.w6()`` and ``.w8()``.
45+
"""
46+
47+
def __init__(self, model: nn.Module, config: Optional[Any] = None) -> None:
48+
from coreai_opt.palettization import KMeansPalettizer
49+
50+
self._palettizer = KMeansPalettizer(model, config)
51+
self._prepared: Optional[nn.Module] = None
52+
53+
def prepare(
54+
self,
55+
example_inputs: Sequence[Any],
56+
sensitivity_path: Optional[str] = None,
57+
num_workers: int = 1,
58+
) -> nn.Module:
59+
"""Cluster the weights and attach the palette parametrizations.
60+
61+
Args:
62+
example_inputs: Sample inputs used to trace the model.
63+
sensitivity_path: Sensitivities saved by a previous
64+
:meth:`calibration_mode` run, to weight the clustering.
65+
num_workers: Parallelism for clustering, which dominates the cost
66+
on large models.
67+
"""
68+
self._prepared = self._palettizer.prepare(
69+
tuple(example_inputs),
70+
sensitivity_path=sensitivity_path,
71+
num_workers=num_workers,
72+
)
73+
return self._prepared
74+
75+
def _require_prepared(self, method: str) -> None:
76+
if self._prepared is None:
77+
raise RuntimeError(f"Call prepare() before {method}().")
78+
79+
def calibration_mode(self, *, loss_fn: Callable, sensitivity_path=None):
80+
"""Context manager for sensitivity-weighted clustering (SqueezeLLM).
81+
82+
Optional, and unlike the quantizer's calibration this is not needed for
83+
a plain run: k-means clusters the weights, which requires no data.
84+
Supplying a loss function collects squared gradients as per-element
85+
sensitivities so clustering favors the weights that matter most. The
86+
block must call ``step(output, target)`` at least once.
87+
88+
There is no ``training_mode`` counterpart: ``KMeansPalettizer`` does not
89+
support training-time compression.
90+
"""
91+
self._require_prepared("calibration_mode")
92+
return self._palettizer.calibration_mode(
93+
loss_fn=loss_fn, sensitivity_path=sensitivity_path
94+
)
95+
96+
def finalize(self, mmap_dir=None) -> nn.Module:
97+
"""Produce the export-ready palettized model.
98+
99+
Returns an eager ``nn.Module``, not a graph: the palette surfaces as a
100+
``lut_to_dense`` op when the result is exported.
101+
102+
Args:
103+
mmap_dir: Directory to memory-map large palettes through, rather
104+
than holding them in memory.
105+
"""
106+
from coreai_opt.common import ExportBackend
107+
108+
self._require_prepared("finalize")
109+
finalized = self._palettizer.finalize(
110+
backend=ExportBackend.CoreAI, mmap_dir=mmap_dir
111+
)
112+
# coreai_opt clears the model's prepared marker, so a second call would
113+
# otherwise pass this guard and fail deeper with a worse message.
114+
self._prepared = None
115+
return finalized
Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""K-means palettization through the full ExecuTorch lowering path.
8+
9+
Palettization compresses a weight into a lookup table plus an index per
10+
element, so the observables differ from quantization: a ``lut_to_dense`` op and
11+
``lut``/``indices`` state, rather than a scale, zero-point and quantized
12+
weight.
13+
"""
14+
15+
import tempfile
16+
import unittest
17+
from dataclasses import dataclass
18+
from pathlib import Path
19+
20+
import torch
21+
import torch.nn as nn
22+
23+
from executorch.backends.apple.coreai import (
24+
CoreAIPartitioner,
25+
get_default_compile_config,
26+
get_default_passes,
27+
)
28+
from executorch.backends.apple.coreai.palettizer import CoreAIPalettizer
29+
from executorch.exir import to_edge_transform_and_lower
30+
31+
32+
def _model():
33+
"""Deterministic, so two builds can be compared to each other."""
34+
torch.manual_seed(0)
35+
return nn.Sequential(nn.Linear(64, 64), nn.ReLU(), nn.Linear(64, 64)).eval()
36+
37+
38+
def _op_count(exported_program, prefix):
39+
"""How many call_function targets start with ``prefix``.
40+
41+
Matched by prefix because targets carry an overload suffix, e.g.
42+
``lut_to_dense.default``.
43+
"""
44+
return sum(
45+
1
46+
for node in exported_program.graph.nodes
47+
if node.op == "call_function"
48+
and getattr(node.target, "__name__", str(node.target)).startswith(prefix)
49+
)
50+
51+
52+
def _first_lut(finalized):
53+
"""The first palette in a finalized model, for comparing clusterings."""
54+
for name, tensor in finalized.state_dict().items():
55+
if name.endswith(".lut"):
56+
return tensor
57+
raise AssertionError("no palette in the finalized model")
58+
59+
60+
@dataclass(frozen=True)
61+
class PalettizeCase:
62+
"""One palettization preset and the palette it should produce."""
63+
64+
name: str
65+
preset: str
66+
n_bits: int
67+
68+
@property
69+
def palette_entries(self) -> int:
70+
return 2**self.n_bits
71+
72+
def config(self):
73+
from coreai_opt.palettization import KMeansPalettizerConfig
74+
75+
return getattr(KMeansPalettizerConfig.presets, self.preset)()
76+
77+
78+
class PalettizationTest(unittest.TestCase):
79+
"""The presets Apple ships: w4, w6 and w8."""
80+
81+
CASES = (
82+
PalettizeCase(name="w4", preset="w4", n_bits=4),
83+
PalettizeCase(name="w6", preset="w6", n_bits=6),
84+
PalettizeCase(name="w8", preset="w8", n_bits=8),
85+
)
86+
87+
def setUp(self):
88+
self.example_inputs = (torch.randn(2, 64),)
89+
90+
def _finalize(self, case):
91+
palettizer = CoreAIPalettizer(_model(), case.config())
92+
palettizer.prepare(self.example_inputs)
93+
return palettizer.finalize()
94+
95+
def _export(self, case):
96+
return torch.export.export(self._finalize(case), self.example_inputs)
97+
98+
def _pte_size(self, case):
99+
lowered = to_edge_transform_and_lower(
100+
self._export(case),
101+
transform_passes=get_default_passes(),
102+
partitioner=[CoreAIPartitioner()],
103+
compile_config=get_default_compile_config(),
104+
)
105+
return len(bytes(lowered.to_executorch().buffer))
106+
107+
def test_finalize_returns_an_eager_module(self):
108+
"""Unlike the quantizer's convert(), which returns an fx graph."""
109+
finalized = self._finalize(self.CASES[0])
110+
self.assertIsInstance(finalized, nn.Module)
111+
self.assertFalse(hasattr(finalized, "graph"))
112+
113+
def test_weights_become_a_lookup_table(self):
114+
for case in self.CASES:
115+
with self.subTest(case.name):
116+
state = self._export(case).state_dict
117+
luts = [k for k in state if k.endswith(".lut")]
118+
indices = [k for k in state if k.endswith(".indices")]
119+
self.assertTrue(luts, f"no palette in {sorted(state)}")
120+
self.assertEqual(len(luts), len(indices))
121+
# The dense weight is emptied out in favour of the palette.
122+
originals = [
123+
v for k, v in state.items() if k.endswith("weight.original")
124+
]
125+
self.assertEqual(
126+
len(originals),
127+
len(luts),
128+
f"expected one emptied weight per palette in {sorted(state)}",
129+
)
130+
self.assertTrue(all(v.numel() == 0 for v in originals))
131+
132+
def test_palette_size_follows_n_bits(self):
133+
"""A k-bit palette holds 2**k entries; guards a silently ignored preset.
134+
135+
Every palette is checked, not just the first: the model has two linears,
136+
and a preset reaching only one of them would still be wrong.
137+
"""
138+
for case in self.CASES:
139+
with self.subTest(case.name):
140+
state = self._export(case).state_dict
141+
luts = [v for k, v in state.items() if k.endswith(".lut")]
142+
self.assertTrue(luts, "no palette found")
143+
for lut in luts:
144+
self.assertEqual(lut.shape[-2], case.palette_entries)
145+
146+
def test_export_emits_lut_to_dense(self):
147+
for case in self.CASES:
148+
with self.subTest(case.name):
149+
self.assertGreater(_op_count(self._export(case), "lut_to_dense"), 0)
150+
151+
def test_lowers_through_to_executorch(self):
152+
for case in self.CASES:
153+
with self.subTest(case.name):
154+
self.assertGreater(self._pte_size(case), 0)
155+
156+
def test_narrower_palettes_produce_smaller_programs(self):
157+
sizes = {case.name: self._pte_size(case) for case in self.CASES}
158+
self.assertLess(sizes["w4"], sizes["w6"], sizes)
159+
self.assertLess(sizes["w6"], sizes["w8"], sizes)
160+
161+
162+
class PalettizerApiTest(unittest.TestCase):
163+
def test_finalize_before_prepare_is_rejected(self):
164+
palettizer = CoreAIPalettizer(_model())
165+
with self.assertRaisesRegex(RuntimeError, "prepare"):
166+
palettizer.finalize()
167+
168+
def test_calibration_before_prepare_is_rejected(self):
169+
palettizer = CoreAIPalettizer(_model())
170+
with self.assertRaisesRegex(RuntimeError, "before calibration_mode"):
171+
palettizer.calibration_mode(loss_fn=lambda *_: torch.tensor(0.0))
172+
173+
def test_finalize_twice_is_rejected(self):
174+
"""coreai_opt clears its own prepared marker, so the guard must too."""
175+
palettizer = CoreAIPalettizer(_model(), PalettizationTest.CASES[0].config())
176+
palettizer.prepare((torch.randn(2, 64),))
177+
palettizer.finalize()
178+
with self.assertRaisesRegex(RuntimeError, "prepare"):
179+
palettizer.finalize()
180+
181+
def test_no_training_mode(self):
182+
"""KMeansPalettizer does not support training-time compression.
183+
184+
Forwarding to it would return a context manager that raises only on
185+
entry, advertising a capability that cannot work.
186+
"""
187+
self.assertFalse(hasattr(CoreAIPalettizer, "training_mode"))
188+
189+
def test_calibration_is_not_required(self):
190+
"""Plain k-means clusters weights, so it needs no calibration data.
191+
192+
The quantizer's calibration collects activation ranges and is
193+
mandatory; here it only supplies sensitivity weights.
194+
"""
195+
palettizer = CoreAIPalettizer(_model(), PalettizationTest.CASES[0].config())
196+
palettizer.prepare((torch.randn(2, 64),))
197+
self.assertIsInstance(palettizer.finalize(), nn.Module)
198+
199+
def test_sensitivity_calibration_changes_the_palette(self):
200+
"""Weighted k-means must actually use the collected sensitivities."""
201+
sample = (torch.randn(2, 64),)
202+
config = PalettizationTest.CASES[0].config()
203+
204+
plain = CoreAIPalettizer(_model(), config)
205+
plain.prepare(sample)
206+
plain_lut = _first_lut(plain.finalize())
207+
208+
with tempfile.TemporaryDirectory() as d:
209+
path = str(Path(d) / "sensitivity")
210+
weighted = CoreAIPalettizer(_model(), config)
211+
prepared = weighted.prepare(sample)
212+
with weighted.calibration_mode(
213+
loss_fn=lambda out, target: (out - target).pow(2).mean(),
214+
sensitivity_path=path,
215+
) as calibrate:
216+
calibrate.step(prepared(*sample), torch.randn(2, 64))
217+
weighted_lut = _first_lut(weighted.finalize())
218+
219+
self.assertEqual(plain_lut.shape, weighted_lut.shape)
220+
self.assertFalse(
221+
torch.equal(plain_lut, weighted_lut),
222+
"sensitivity-weighted clustering produced an identical palette",
223+
)
224+
225+
226+
if __name__ == "__main__":
227+
unittest.main()

0 commit comments

Comments
 (0)