Skip to content

Commit 3087563

Browse files
ssjiaSS-JIA
authored andcommitted
[ET-VK][q8ta] Add q8ta_linear operator for int8 quantized linear
Pull Request resolved: pytorch#17565 Add a new q8ta_linear operator that performs fully quantized int8 linear (matmul + bias) with per-tensor activation quantization and per-channel weight quantization, producing int8 output. This enables back-to-back quantized linear layers without intermediate dequantize/quantize steps. The operator reuses the existing tiled int8 linear GLSL headers (input/weight tile loading, int8 dot product accumulation, weight scales/sums/bias loading) and adds output quantization via quantize_and_pack to produce packed int8 output. The fusion pass in quantized_linear.py detects the q→dq→linear→q pattern (where the output quantize node comes from a subsequent quantized op's input) and fuses it into a single q8ta_linear call. This diff was authored with Claude. ghstack-source-id: 343460521 @exported-using-ghexport Differential Revision: [D93768642](https://our.internmc.facebook.com/intern/diff/D93768642/)
1 parent 2244dc3 commit 3087563

14 files changed

Lines changed: 1058 additions & 4 deletions

‎backends/vulkan/custom_ops_lib.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,71 @@ def linear_q8ta_q8csw(
356356
lib.impl(name, linear_q8ta_q8csw, "CompositeExplicitAutograd")
357357
qa_q8csw_linear = getattr(getattr(torch.ops, namespace), name)
358358

359+
##################
360+
## q8ta_linear ##
361+
##################
362+
363+
364+
def q8ta_linear(
365+
x: torch.Tensor,
366+
input_scale: float,
367+
input_zero_point: int,
368+
weights: torch.Tensor,
369+
weight_sums: torch.Tensor,
370+
weight_scales: torch.Tensor,
371+
output_scale: float,
372+
output_zero_point: int,
373+
bias: Optional[torch.Tensor] = None,
374+
activation: str = "none",
375+
):
376+
weight_zeros = torch.zeros_like(weight_scales, dtype=torch.int32)
377+
weights = torch.ops.quantized_decomposed.dequantize_per_channel(
378+
weights,
379+
weight_scales,
380+
weight_zeros,
381+
0,
382+
-127,
383+
127,
384+
torch.int8,
385+
)
386+
387+
x = torch.ops.quantized_decomposed.dequantize_per_tensor(
388+
x, input_scale, input_zero_point, -128, 127, x.dtype
389+
)
390+
391+
out = torch.nn.functional.linear(x, weights)
392+
if bias is not None:
393+
out = out + bias
394+
395+
if activation == "relu":
396+
out = torch.nn.functional.relu(out)
397+
398+
out = torch.ops.quantized_decomposed.quantize_per_tensor(
399+
out, output_scale, output_zero_point, -128, 127, torch.int8
400+
)
401+
402+
return out
403+
404+
405+
name = "q8ta_linear"
406+
lib.define(
407+
f"""
408+
{name}(
409+
Tensor x,
410+
float input_scale,
411+
int input_zero_point,
412+
Tensor weights,
413+
Tensor weight_sums,
414+
Tensor weight_scales,
415+
float output_scale,
416+
int output_zero_point,
417+
Tensor? bias = None,
418+
str activation = "none") -> Tensor
419+
"""
420+
)
421+
lib.impl(name, q8ta_linear, "CompositeExplicitAutograd")
422+
q8ta_linear_op = getattr(getattr(torch.ops, namespace), name)
423+
359424
###################
360425
## q8ta_conv2d_* ##
361426
###################

‎backends/vulkan/op_registry.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -830,6 +830,34 @@ def register_q8ta_conv2d_ops():
830830
)
831831

832832

833+
# =============================================================================
834+
# Q8taLinear.cpp
835+
# =============================================================================
836+
837+
838+
@update_features(exir_ops.edge.et_vk.q8ta_linear.default)
839+
def register_q8ta_linear():
840+
return OpFeatures(
841+
inputs_storage=[
842+
utils.PACKED_INT8_4H4W_BUFFER, # input
843+
utils.NO_STORAGE, # input_scale (non tensor)
844+
utils.NO_STORAGE, # input_zero_point (non tensor)
845+
utils.NO_STORAGE, # weight (prepacked)
846+
utils.NO_STORAGE, # weight_sums (prepacked)
847+
utils.NO_STORAGE, # weight_scales (prepacked)
848+
utils.NO_STORAGE, # output_scale (non tensor)
849+
utils.NO_STORAGE, # output_zero_point (non tensor)
850+
utils.NO_STORAGE, # bias (prepacked)
851+
utils.NO_STORAGE, # activation (non tensor)
852+
],
853+
outputs_storage=[
854+
utils.PACKED_INT8_4H4W_BUFFER,
855+
],
856+
supports_resize=False,
857+
supports_prepacking=True,
858+
)
859+
860+
833861
# =============================================================================
834862
# SDPA.cpp
835863
# =============================================================================

‎backends/vulkan/patterns/quantized_linear.py‎

Lines changed: 88 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131

3232

3333
class QuantizedLinearMatch(PatternMatch):
34-
def __init__(self, mm_node: torch.fx.Node) -> None:
34+
def __init__(self, mm_node: torch.fx.Node) -> None: # noqa: C901
3535
self.anchor_node = mm_node
3636
self.match_found = False
3737
self.all_nodes = [self.anchor_node]
@@ -111,10 +111,17 @@ def __init__(self, mm_node: torch.fx.Node) -> None:
111111
self.bias_node = None
112112
if self.anchor_node.target == exir_ops.edge.aten.addmm.default:
113113
self.bias_node, arg_chain = utils.trace_args_until_placeholder(
114-
self.anchor_node.args[2]
114+
self.anchor_node.args[0]
115115
)
116116
assert self.bias_node is not None
117117
self.all_nodes.extend(arg_chain)
118+
elif self.anchor_node.target == exir_ops.edge.aten.linear.default:
119+
if len(self.anchor_node.args) > 2 and self.anchor_node.args[2] is not None:
120+
self.bias_node, arg_chain = utils.trace_args_until_placeholder(
121+
self.anchor_node.args[2]
122+
)
123+
if self.bias_node is not None:
124+
self.all_nodes.extend(arg_chain)
118125

119126
# If input is not quantized, then we are done
120127
if self.quantize_input_node is None:
@@ -143,11 +150,36 @@ def __init__(self, mm_node: torch.fx.Node) -> None:
143150
]
144151
)
145152

153+
# Check if the output is also quantized (q → dq → linear → q pattern)
154+
# Also handle fused linear+relu (q → dq → linear → relu → q pattern)
155+
self.quantize_output_node = None
156+
self.output_scales_node = None
157+
self.output_zeros_node = None
158+
self.relu_node = None
159+
if len(self.output_node.users) == 1:
160+
cur_node = list(self.output_node.users)[0]
161+
if cur_node.target == exir_ops.edge.aten.relu.default:
162+
self.relu_node = cur_node
163+
if len(cur_node.users) == 1:
164+
cur_node = list(cur_node.users)[0]
165+
else:
166+
cur_node = None
167+
if cur_node is not None and utils.is_quant_node(cur_node):
168+
self.quantize_output_node = cur_node
169+
self.output_scales_node = self.quantize_output_node.args[1]
170+
self.output_zeros_node = self.quantize_output_node.args[2]
171+
146172
self.match_found = True
147173

148174
def is_weight_only_quantized(self) -> bool:
149175
return self.quantize_input_node is None
150176

177+
def has_output_quantization(self) -> bool:
178+
return (
179+
hasattr(self, "quantize_output_node")
180+
and self.quantize_output_node is not None
181+
)
182+
151183
def is_weight_pergroup_quantized(self) -> bool:
152184
weight_shape = self.weight_node.meta["val"].shape
153185
scales_shape = self.weight_scales_node.meta["val"].shape
@@ -454,6 +486,49 @@ def make_linear_q8ta_q8csw_custom_op(
454486
match.output_node.replace_all_uses_with(qlinear_node)
455487

456488

489+
def make_q8ta_linear_custom_op(
490+
ep: ExportedProgram,
491+
graph_module: torch.fx.GraphModule,
492+
match: QuantizedLinearMatch,
493+
weight_tensor: torch.Tensor,
494+
):
495+
first_graph_node = list(graph_module.graph.nodes)[0]
496+
with graph_module.graph.inserting_before(first_graph_node):
497+
weight_tensor_name = utils.get_tensor_name(ep, match.weight_node)
498+
sum_per_output_channel = weight_tensor.sum(dim=1).to(torch.int32).contiguous()
499+
sums_name = weight_tensor_name + "_sums"
500+
sums_name = sums_name.replace(".", "_")
501+
502+
weight_sums_node = create_constant_placeholder(
503+
exp_program=ep,
504+
graph=graph_module.graph,
505+
kind=InputKind.PARAMETER,
506+
name=sums_name,
507+
data=sum_per_output_channel,
508+
)
509+
510+
with graph_module.graph.inserting_before(match.output_node):
511+
qlinear_node = graph_module.graph.create_node(
512+
"call_function",
513+
exir_ops.edge.et_vk.q8ta_linear.default,
514+
args=(
515+
match.quantize_input_node,
516+
match.input_scales_node,
517+
match.input_zeros_node,
518+
match.weight_node,
519+
weight_sums_node,
520+
match.weight_scales_node,
521+
match.output_scales_node,
522+
match.output_zeros_node,
523+
match.bias_node,
524+
"relu" if match.relu_node is not None else "none",
525+
),
526+
)
527+
528+
qlinear_node.meta["val"] = match.quantize_output_node.meta["val"]
529+
match.quantize_output_node.replace_all_uses_with(qlinear_node)
530+
531+
457532
@register_pattern_replacement("quantized_linear")
458533
def replace_quantized_linear_patterns(
459534
ep: ExportedProgram,
@@ -472,11 +547,20 @@ def replace_quantized_linear_patterns(
472547
weight_zeros_tensor = get_param_tensor(ep, match.weight_zeros_node)
473548
assert weight_zeros_tensor is not None
474549

475-
# Biases not supported at the moment
550+
# Route to appropriate custom op.
551+
# q8ta_linear supports bias, so check it first before the bias guard.
552+
if (
553+
match.is_input_static_per_tensor_quantized()
554+
and match.is_weight_perchannel_quantized()
555+
and match.has_output_quantization()
556+
):
557+
make_q8ta_linear_custom_op(ep, graph_module, match, weight_tensor)
558+
return
559+
560+
# Remaining ops do not support bias
476561
if match.bias_node is not None:
477562
return
478563

479-
# Route to appropriate custom op
480564
if (
481565
match.is_weight_only_quantized()
482566
and match.is_weight_pergroup_quantized()

0 commit comments

Comments
 (0)