Skip to content

Commit 910f816

Browse files
authored
Add packed W4 execution to ROCm AOTI (pytorch#21946)
Nibble-pack symmetric W4 weights and lower packed linears to Triton W4A16 matmul. Add an off-by-default BF16-rounded single-row matvec while preserving existing matvec behavior, and run ROCm execution tests through the native runner. Retain upstream AOTI header precompilation after cold-cache coverage.
1 parent 5b3b6ff commit 910f816

8 files changed

Lines changed: 457 additions & 21 deletions

File tree

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Nibble-packed W4/BF16 weights for the AOTI Triton matmul path."""
8+
9+
import torch
10+
import torch.nn as nn
11+
from executorch.backends.cuda.triton.kernels.int4_matmul import (
12+
int4_matmul,
13+
int4_matvec_bf16,
14+
)
15+
from torchao.utils import TorchAOBaseTensor
16+
17+
18+
class AotiPackedInt4Tensor(TorchAOBaseTensor):
19+
"""Symmetric groupwise INT4 weight consumed by AOTI Triton kernels.
20+
21+
Linears use ``triton::int4_matmul`` by default; the opt-in fixed-shape path
22+
uses ``triton::int4_matvec_bf16``.
23+
"""
24+
25+
tensor_data_names = ["qdata", "scale"]
26+
tensor_attribute_names = ["group_size", "orig_dtype", "use_matvec"]
27+
28+
def __new__(cls, qdata, scale, group_size, orig_dtype, use_matvec=False):
29+
shape = (qdata.shape[0], qdata.shape[1] * 2)
30+
self = torch.Tensor._make_wrapper_subclass(
31+
cls,
32+
shape,
33+
dtype=orig_dtype,
34+
device=qdata.device,
35+
requires_grad=False,
36+
)
37+
self.qdata = qdata
38+
self.scale = scale
39+
self.group_size = group_size
40+
self.orig_dtype = orig_dtype
41+
self.use_matvec = use_matvec
42+
return self
43+
44+
@classmethod
45+
def from_intx_unpacked_to_int8_tensor(
46+
cls, weight: torch.Tensor, use_matvec: bool = False
47+
) -> "AotiPackedInt4Tensor":
48+
from torchao.quantization import IntxUnpackedToInt8Tensor
49+
50+
if not isinstance(weight, IntxUnpackedToInt8Tensor):
51+
raise TypeError(
52+
"AotiPackedInt4Tensor requires IntxUnpackedToInt8Tensor, got "
53+
f"{type(weight).__name__}"
54+
)
55+
if weight.target_dtype != torch.int4:
56+
raise ValueError("AotiPackedInt4Tensor requires target_dtype=torch.int4")
57+
if weight.activation_quantization is not None:
58+
raise ValueError("AotiPackedInt4Tensor only supports weight-only INT4")
59+
if weight.dtype != torch.bfloat16:
60+
raise ValueError("AotiPackedInt4Tensor requires BF16 activations")
61+
if weight.qdata.ndim != 2 or weight.block_size[0] != 1:
62+
raise ValueError("AotiPackedInt4Tensor requires 2D groupwise weights")
63+
if torch.count_nonzero(weight.zero_point).item() != 0:
64+
raise ValueError("AotiPackedInt4Tensor requires symmetric zero points")
65+
group_size = int(weight.block_size[-1])
66+
rows, columns = weight.qdata.shape
67+
if columns % 2 != 0 or columns % group_size != 0:
68+
raise ValueError(
69+
"AotiPackedInt4Tensor requires the input dimension to be "
70+
"divisible by two and the group size"
71+
)
72+
if weight.scale.shape != (rows, columns // group_size):
73+
raise ValueError("AotiPackedInt4Tensor received incompatible scales")
74+
75+
qdata = (weight.qdata + 8).to(torch.uint8)
76+
qdata = (qdata[..., ::2] | (qdata[..., 1::2] << 4)).to(torch.int8)
77+
return cls(
78+
qdata.contiguous(),
79+
weight.scale.contiguous(),
80+
group_size,
81+
weight.dtype,
82+
use_matvec,
83+
)
84+
85+
def dequantize(self, output_dtype=None):
86+
dtype = output_dtype or self.orig_dtype
87+
packed = self.qdata.to(torch.uint8)
88+
low = (packed & 0x0F).to(torch.float32) - 8.0
89+
high = ((packed >> 4) & 0x0F).to(torch.float32) - 8.0
90+
values = torch.stack([low, high], dim=-1).reshape(self.shape)
91+
scale = self.scale.float().repeat_interleave(self.group_size, dim=-1)
92+
return (values * scale).to(dtype)
93+
94+
def to(self, *args, **kwargs):
95+
kwargs = self._get_to_kwargs(*args, **kwargs)
96+
device = kwargs.pop("device")
97+
dtype = kwargs.pop("dtype")
98+
if dtype != torch.bfloat16:
99+
raise ValueError("AotiPackedInt4Tensor requires BF16 activations")
100+
return AotiPackedInt4Tensor(
101+
self.qdata.to(device),
102+
self.scale.to(device=device, dtype=dtype),
103+
self.group_size,
104+
dtype,
105+
self.use_matvec,
106+
)
107+
108+
__torch_function__ = torch._C._disabled_torch_function_impl
109+
110+
111+
implements = AotiPackedInt4Tensor.implements
112+
113+
114+
@implements([torch.ops.aten.linear.default])
115+
def _(func, types, args, kwargs):
116+
input_tensor, weight = args[0], args[1]
117+
bias = args[2] if len(args) > 2 else None
118+
input_shape = input_tensor.shape
119+
input_2d = input_tensor.reshape(-1, input_shape[-1])
120+
rows = input_2d.shape[0]
121+
if isinstance(rows, int) and rows == 1 and weight.use_matvec:
122+
output = int4_matvec_bf16(
123+
input_2d,
124+
weight.qdata,
125+
weight.scale,
126+
weight.group_size,
127+
)
128+
else:
129+
output = int4_matmul(
130+
input_2d,
131+
weight.qdata,
132+
weight.scale,
133+
weight.group_size,
134+
)
135+
output = output.reshape(*input_shape[:-1], weight.shape[0])
136+
return output if bias is None else output + bias
137+
138+
139+
@implements([torch.ops.aten.detach.default, torch.ops.aten.alias.default])
140+
def _(func, types, args, kwargs):
141+
return args[0]
142+
143+
144+
@implements([torch.ops.aten.t.default])
145+
def _(func, types, args, kwargs):
146+
return args[0].dequantize().t()
147+
148+
149+
@implements([torch.ops.aten._to_copy.default])
150+
def _(func, types, args, kwargs):
151+
return args[0].to(**kwargs)
152+
153+
154+
def pack_int4_weights_for_aoti(module: nn.Module, *, use_matvec: bool = False) -> int:
155+
"""Pack symmetric TorchAO INT4 linear weights in ``module`` in place."""
156+
from torchao.quantization import IntxUnpackedToInt8Tensor
157+
158+
packed = 0
159+
for child in module.modules():
160+
if not isinstance(child, nn.Linear):
161+
continue
162+
weight = child.weight
163+
if not isinstance(weight, IntxUnpackedToInt8Tensor):
164+
continue
165+
if weight.target_dtype != torch.int4:
166+
continue
167+
child.weight = nn.Parameter(
168+
AotiPackedInt4Tensor.from_intx_unpacked_to_int8_tensor(
169+
weight,
170+
use_matvec=use_matvec,
171+
),
172+
requires_grad=False,
173+
)
174+
packed += 1
175+
return packed
176+
177+
178+
torch.serialization.add_safe_globals([AotiPackedInt4Tensor])
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import unittest
8+
9+
import torch
10+
import torch.nn as nn
11+
from executorch.backends.cuda.aoti_packed_int4_tensor import (
12+
AotiPackedInt4Tensor,
13+
pack_int4_weights_for_aoti,
14+
)
15+
from executorch.extension.llm.export.quantize import quantize_model_
16+
17+
18+
def _make_quantized_linear(device="cpu"):
19+
module = nn.Linear(64, 32, bias=False, dtype=torch.bfloat16, device=device)
20+
quantize_model_(
21+
module,
22+
qlinear_config="4w",
23+
qlinear_group_size=32,
24+
)
25+
return module
26+
27+
28+
class TestAotiPackedInt4Tensor(unittest.TestCase):
29+
def test_pack_halves_weight_storage(self):
30+
module = _make_quantized_linear()
31+
unpacked = module.weight
32+
expected = unpacked.dequantize()
33+
34+
self.assertEqual(pack_int4_weights_for_aoti(module), 1)
35+
packed = module.weight
36+
37+
self.assertIsInstance(packed, AotiPackedInt4Tensor)
38+
self.assertEqual(packed.qdata.shape, (32, 32))
39+
torch.testing.assert_close(packed.dequantize(), expected)
40+
41+
@unittest.skipUnless(torch.cuda.is_available(), "GPU required")
42+
def test_eager_linear_uses_packed_weight(self):
43+
torch.manual_seed(42)
44+
module = _make_quantized_linear("cuda")
45+
inputs = torch.randn(7, 64, dtype=torch.bfloat16, device="cuda")
46+
expected = module(inputs)
47+
48+
self.assertEqual(pack_int4_weights_for_aoti(module), 1)
49+
actual = module(inputs)
50+
51+
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.02)
52+
53+
@unittest.skipUnless(torch.cuda.is_available(), "GPU required")
54+
def test_eager_single_row_uses_packed_weight(self):
55+
torch.manual_seed(42)
56+
module = _make_quantized_linear("cuda")
57+
inputs = torch.randn(1, 64, dtype=torch.bfloat16, device="cuda")
58+
expected = module(inputs)
59+
60+
self.assertEqual(pack_int4_weights_for_aoti(module, use_matvec=True), 1)
61+
actual = module(inputs)
62+
63+
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.02)
64+
65+
@unittest.skipUnless(torch.cuda.is_available(), "GPU required")
66+
def test_export_contains_triton_int4_matmul(self):
67+
module = _make_quantized_linear("cuda")
68+
pack_int4_weights_for_aoti(module)
69+
inputs = (torch.randn(4, 64, dtype=torch.bfloat16, device="cuda"),)
70+
71+
exported = torch.export.export(module, inputs, strict=True).run_decompositions(
72+
{}
73+
)
74+
75+
self.assertIn("triton.int4_matmul.default", exported.graph_module.code)
76+
self.assertNotIn("triton.int4_matvec_bf16.default", exported.graph_module.code)
77+
self.assertNotIn("dequantize_affine", exported.graph_module.code)
78+
79+
@unittest.skipUnless(torch.cuda.is_available(), "GPU required")
80+
def test_single_row_export_defaults_to_triton_int4_matmul(self):
81+
module = _make_quantized_linear("cuda")
82+
pack_int4_weights_for_aoti(module)
83+
inputs = (torch.randn(1, 64, dtype=torch.bfloat16, device="cuda"),)
84+
85+
exported = torch.export.export(module, inputs, strict=True).run_decompositions(
86+
{}
87+
)
88+
89+
self.assertIn("triton.int4_matmul.default", exported.graph_module.code)
90+
self.assertNotIn("triton.int4_matvec_bf16.default", exported.graph_module.code)
91+
self.assertNotIn("dequantize_affine", exported.graph_module.code)
92+
93+
@unittest.skipUnless(torch.cuda.is_available(), "GPU required")
94+
def test_single_row_export_can_select_triton_int4_matvec(self):
95+
module = _make_quantized_linear("cuda")
96+
pack_int4_weights_for_aoti(module, use_matvec=True)
97+
inputs = (torch.randn(1, 64, dtype=torch.bfloat16, device="cuda"),)
98+
99+
exported = torch.export.export(module, inputs, strict=True).run_decompositions(
100+
{}
101+
)
102+
103+
self.assertIn("triton.int4_matvec_bf16.default", exported.graph_module.code)
104+
self.assertNotIn("triton.int4_matmul.default", exported.graph_module.code)
105+
self.assertNotIn("dequantize_affine", exported.graph_module.code)
106+
107+
108+
if __name__ == "__main__":
109+
unittest.main()

‎backends/cuda/tests/test_int4_matmul.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
dequant_w4_to_bf16,
2424
int4_matmul,
2525
int4_matvec,
26+
int4_matvec_bf16,
2627
)
2728

2829
ATOL = 0.01
@@ -251,6 +252,24 @@ def test_matches_int4_matmul(self):
251252

252253
_assert_snr(self, out_mv, out_mm, "matvec vs matmul")
253254

255+
def test_bf16_weight_rounding_matches_int4_matmul(self):
256+
torch.manual_seed(42)
257+
N, K, gs = 2048, 2048, 128
258+
w = torch.randn(N, K, dtype=torch.bfloat16, device=DEVICE)
259+
packed, scale, _ = _quantize_simple(w, gs)
260+
x = torch.randn(1, K, dtype=torch.bfloat16, device=DEVICE)
261+
262+
out_mv = int4_matvec_bf16(x, packed, scale, gs)
263+
out_mm = int4_matmul(x, packed, scale, gs)
264+
265+
_assert_snr(
266+
self,
267+
out_mv,
268+
out_mm,
269+
"BF16-rounded matvec vs matmul",
270+
threshold_db=80.0,
271+
)
272+
254273

255274
class TestDequantThenMatmul(unittest.TestCase):
256275
"""Tests that dequant + F.linear matches int4_matmul (both paths should agree)."""

0 commit comments

Comments
 (0)