Skip to content

Commit 4a06a4f

Browse files
ssjiaSS-JIA
authored andcommitted
[ET-VK][q8ta] Fix addmm arg indexing in QuantizedLinearMatch
Pull Request resolved: pytorch#17567 QuantizedLinearMatch always used args[1] for the weight and args[0] for the input, which is correct for mm(input, weight) and linear(input, weight, bias?) but wrong for addmm(bias, input, weight) where the weight is at args[2] and the input is at args[1]. This was exposed by a torchao change (D69887498) that added Linear+BatchNorm fusion to prepare_pt2e(). The fusion adds a bias to Linear nodes that previously had none, causing them to decompose to addmm instead of mm in the edge dialect. The pattern matcher then read the input's per-tensor dequantize scale (a float literal) as if it were the weight's per-channel scale (a Node), causing an assertion failure. The fix determines the correct arg indices based on whether the anchor node is addmm. The bias handling at args[0] for addmm was already correct. Authored-by: Claude When two quantized linears are chained (e.g. in the SceneX prediction head), the pattern registry processes them in topological order and applies replacements immediately. The first linear's replacement calls `replace_all_uses_with`, which rewrites the dq node's input from the original quantize op to the new q8ta_linear op. When the second linear is then matched, `maybe_skip_q_dq_arg_chain` traces back through the dq node and finds the q8ta_linear op instead of the original quantize op. The code then extracts scale/zp from args[1]/args[2] of that node, but q8ta_linear has a different args layout than quantize_per_tensor— args[1]/args[2] are the first linear's INPUT scale/zp, not its OUTPUT scale/zp. This causes the second linear to wildly misinterpret its input values, saturating outputs to -128/127. The fix reads input scale/zp from the dq node's args instead of the quantize node's args. The dq node always retains the correct scale/zp because `replace_all_uses_with` only rewrites its input tensor (args[0]), not the scale/zp args. This is both simpler and more robust than special-casing the q8ta_linear args layout. Pull Request resolved: pytorch#17567 QuantizedLinearMatch always used args[1] for the weight and args[0] for the input, which is correct for mm(input, weight) and linear(input, weight, bias?) but wrong for addmm(bias, input, weight) where the weight is at args[2] and the input is at args[1]. This was exposed by a torchao change (D69887498) that added Linear+BatchNorm fusion to prepare_pt2e(). The fusion adds a bias to Linear nodes that previously had none, causing them to decompose to addmm instead of mm in the edge dialect. The pattern matcher then read the input's per-tensor dequantize scale (a float literal) as if it were the weight's per-channel scale (a Node), causing an assertion failure. The fix determines the correct arg indices based on whether the anchor node is addmm. The bias handling at args[0] for addmm was already correct. Authored-by: Claude ghstack-source-id: 343460523 @exported-using-ghexport Differential Revision: [D93768640](https://our.internmc.facebook.com/intern/diff/D93768640/)
1 parent 187dc1d commit 4a06a4f

4 files changed

Lines changed: 245 additions & 50 deletions

File tree

‎backends/vulkan/custom_ops_lib.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,7 @@ def q8ta_linear(
390390

391391
out = torch.nn.functional.linear(x, weights)
392392
if bias is not None:
393-
out = out + bias
393+
out = out + bias[: out.shape[-1]]
394394

395395
if activation == "relu":
396396
out = torch.nn.functional.relu(out)
@@ -455,7 +455,7 @@ def q8ta_linear_gemv(
455455

456456
out = torch.nn.functional.linear(x, weights)
457457
if bias is not None:
458-
out = out + bias
458+
out = out + bias[: out.shape[-1]]
459459

460460
if activation == "relu":
461461
out = torch.nn.functional.relu(out)

‎backends/vulkan/patterns/quantized_linear.py‎

Lines changed: 114 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,14 @@ def __init__(self, mm_node: torch.fx.Node) -> None: # noqa: C901
3636
self.match_found = False
3737
self.all_nodes = [self.anchor_node]
3838

39+
# addmm(bias, mat1, mat2) has a different arg layout than
40+
# mm(mat1, mat2) and linear(input, weight, bias?)
41+
is_addmm = self.anchor_node.target == exir_ops.edge.aten.addmm.default
42+
weight_arg_idx = 2 if is_addmm else 1
43+
input_arg_idx = 1 if is_addmm else 0
44+
3945
const_node, arg_chain = utils.trace_args_until_placeholder(
40-
self.anchor_node.args[1]
46+
self.anchor_node.args[weight_arg_idx]
4147
)
4248

4349
# mat2 is not a constant tensor - no match
@@ -84,26 +90,64 @@ def __init__(self, mm_node: torch.fx.Node) -> None: # noqa: C901
8490
# Identify output node
8591
self.output_node = self.anchor_node
8692

87-
# The implementation has a limitation that output channels must be a
88-
# multiple of 4. This is to ensure that data loads are aligned well with
89-
# texel boundaries. If this is not true, then don't match the pattern.
90-
out_channels = self.output_node.meta["val"].shape[-1]
91-
if out_channels % 4 != 0:
92-
return
93+
# Identify primary input node of the anchor. Due to decomposition of aten.linear
94+
# there may be a view_copy node between the original input tensor to the linear
95+
# op and the actual linear op node.
96+
anchor_primary_input_node = self.anchor_node.args[input_arg_idx]
97+
assert isinstance(anchor_primary_input_node, torch.fx.Node)
98+
99+
# Skip potential view_copy between dq and linear
100+
if utils.is_view_copy_node(anchor_primary_input_node):
101+
self.all_nodes.append(anchor_primary_input_node)
102+
anchor_primary_input_node = anchor_primary_input_node.args[
103+
0
104+
] # pyre-ignore[16]
105+
assert isinstance(anchor_primary_input_node, torch.fx.Node)
106+
107+
# By default, assume that the input tensor is not quantized in any way
108+
self.quantize_input_node = None
109+
self.dequantize_input_node = None
110+
self.pattern_input_node = anchor_primary_input_node
111+
112+
self.input_scales_node = None
113+
self.input_zeros_node = None
114+
115+
scales_arg_idx = 1
116+
zeros_arg_idx = 2
93117

94-
# Identify input node
95-
(
96-
self.fp_input_node,
97-
self.quantize_input_node,
98-
dq_node,
99-
) = utils.maybe_skip_q_dq_arg_chain(self.anchor_node.args[0])
100-
assert self.fp_input_node is not None
101-
self.all_nodes.append(self.fp_input_node)
118+
# If the primary input node comes from a dequantize node, that implies the input
119+
# input tensor is quantized (either statically or dynamically).
120+
if utils.is_dequant_node(anchor_primary_input_node):
121+
# Assume that this is a static quantization pattern; the input to the
122+
# pattern is a statically quantized int8 tensor.
123+
self.dequantize_input_node = anchor_primary_input_node
124+
self.all_nodes.append(self.dequantize_input_node)
125+
input_to_dq_node = self.dequantize_input_node.args[0]
126+
self.pattern_input_node = input_to_dq_node
127+
128+
# torchao dequantize has a slightly different function schema
129+
if (
130+
self.dequantize_input_node.target
131+
== exir_ops.edge.torchao.dequantize_affine.default
132+
):
133+
scales_arg_idx = 2
134+
zeros_arg_idx = 3
135+
136+
self.input_scales_node = self.dequantize_input_node.args[scales_arg_idx]
137+
self.input_zeros_node = self.dequantize_input_node.args[zeros_arg_idx]
138+
139+
# Check for dynamic quantization: input scales are dynamically
140+
# computed via a choose_qparams op
141+
if utils.is_quant_node(input_to_dq_node) and utils.is_dynamic_qscale(
142+
self.input_scales_node
143+
):
144+
self.quantize_input_node = input_to_dq_node
145+
self.pattern_input_node = self.quantize_input_node.args[0]
102146

103147
# The implementation has a limitation that input channels must be a
104148
# multiple of 4. This is to ensure that data loads are aligned well with
105149
# texel boundaries. If this is not true, then don't match the pattern.
106-
in_channels = self.fp_input_node.meta["val"].shape[-1]
150+
in_channels = self.pattern_input_node.meta["val"].shape[-1]
107151
if in_channels % 4 != 0:
108152
return
109153

@@ -124,32 +168,10 @@ def __init__(self, mm_node: torch.fx.Node) -> None: # noqa: C901
124168
self.all_nodes.extend(arg_chain)
125169

126170
# If input is not quantized, then we are done
127-
if self.quantize_input_node is None:
171+
if self.dequantize_input_node is None:
128172
self.match_found = True
129173
return
130174

131-
scales_arg_idx = 1
132-
zeros_arg_idx = 2
133-
134-
# torchao op has a slightly different function schema
135-
if (
136-
self.quantize_input_node.target
137-
== exir_ops.edge.torchao.quantize_affine.default
138-
):
139-
scales_arg_idx = 2
140-
zeros_arg_idx = 3
141-
142-
self.input_scales_node = self.quantize_input_node.args[scales_arg_idx]
143-
self.input_zeros_node = self.quantize_input_node.args[zeros_arg_idx]
144-
145-
assert dq_node is not None
146-
self.all_nodes.extend(
147-
[
148-
self.quantize_input_node,
149-
dq_node,
150-
]
151-
)
152-
153175
# Check if the output is also quantized (q → dq → linear → q pattern)
154176
# Also handle fused linear+relu (q → dq → linear → relu → q pattern)
155177
self.quantize_output_node = None
@@ -172,7 +194,7 @@ def __init__(self, mm_node: torch.fx.Node) -> None: # noqa: C901
172194
self.match_found = True
173195

174196
def is_weight_only_quantized(self) -> bool:
175-
return self.quantize_input_node is None
197+
return self.dequantize_input_node is None
176198

177199
def has_output_quantization(self) -> bool:
178200
return (
@@ -204,15 +226,15 @@ def is_weight_perchannel_quantized(self) -> bool:
204226
return scales_shape[0] == weight_shape[-2]
205227

206228
def is_input_static_per_tensor_quantized(self) -> bool:
207-
if self.quantize_input_node is None:
229+
if self.dequantize_input_node is None:
208230
return False
209231

210232
# For static quantization per tensor quantization, the scales and zeros
211233
# are scalars.
212234
return isinstance(self.input_scales_node, float)
213235

214236
def is_input_dynamic_perchannel_quantized(self) -> bool:
215-
if self.quantize_input_node is None:
237+
if self.dequantize_input_node is None:
216238
return False
217239

218240
if not isinstance(self.input_scales_node, torch.fx.Node):
@@ -228,7 +250,7 @@ def is_input_dynamic_perchannel_quantized(self) -> bool:
228250
return False
229251

230252
scales_shape = self.input_scales_node.meta["val"].shape
231-
input_shape = self.fp_input_node.meta["val"].shape
253+
input_shape = self.pattern_input_node.meta["val"].shape
232254

233255
return input_shape[-2] == scales_shape[-1]
234256

@@ -366,7 +388,7 @@ def make_linear_q4gsw_op(
366388
"call_function",
367389
exir_ops.edge.et_vk.linear_q4gsw.default,
368390
args=(
369-
match.fp_input_node,
391+
match.pattern_input_node,
370392
match.weight_node,
371393
match.weight_scales_node,
372394
group_size,
@@ -430,7 +452,7 @@ def make_linear_dq8ca_q4gsw_op(
430452
"call_function",
431453
exir_ops.edge.et_vk.linear_dq8ca_q4gsw.default,
432454
args=(
433-
match.fp_input_node,
455+
match.pattern_input_node,
434456
match.input_scales_node,
435457
match.input_zeros_node,
436458
match.weight_node,
@@ -450,12 +472,34 @@ def make_linear_q8ta_q8csw_custom_op(
450472
match: QuantizedLinearMatch,
451473
weight_tensor: torch.Tensor,
452474
):
475+
# Pad weight_scales to multiple of 4 so GPU shader reads don't go OOB
476+
weight_scales_tensor = get_param_tensor(ep, match.weight_scales_node)
477+
assert weight_scales_tensor is not None
478+
utils.align_width_and_update_state_dict(
479+
ep, match.weight_scales_node, weight_scales_tensor
480+
)
481+
482+
# Pad bias to multiple of 4 if present
483+
if match.bias_node is not None:
484+
bias_tensor = get_param_tensor(ep, match.bias_node)
485+
if bias_tensor is not None:
486+
utils.align_width_and_update_state_dict(ep, match.bias_node, bias_tensor)
487+
453488
first_graph_node = list(graph_module.graph.nodes)[0]
454489
with graph_module.graph.inserting_before(first_graph_node):
455490
weight_tensor_name = utils.get_tensor_name(ep, match.weight_node)
456491
# Pre-compute the weight sums which are needed to apply activation zero point
457492
# when using integer accumulation.
458493
sum_per_output_channel = weight_tensor.sum(dim=1).to(torch.int32).contiguous()
494+
495+
# Pad weight sums to align OC to multiple of 4
496+
oc = sum_per_output_channel.shape[0]
497+
if oc % 4 != 0:
498+
num_padding = 4 - (oc % 4)
499+
sum_per_output_channel = F.pad(
500+
sum_per_output_channel, (0, num_padding)
501+
).contiguous()
502+
459503
sums_name = weight_tensor_name + "_sums"
460504
# Sanitize the name
461505
sums_name = sums_name.replace(".", "_")
@@ -473,7 +517,7 @@ def make_linear_q8ta_q8csw_custom_op(
473517
"call_function",
474518
exir_ops.edge.et_vk.linear_q8ta_q8csw.default,
475519
args=(
476-
match.fp_input_node,
520+
match.pattern_input_node,
477521
match.input_scales_node,
478522
match.input_zeros_node,
479523
match.weight_node,
@@ -492,10 +536,32 @@ def make_q8ta_linear_custom_op(
492536
match: QuantizedLinearMatch,
493537
weight_tensor: torch.Tensor,
494538
):
539+
# Pad weight_scales to multiple of 4 so GPU shader reads don't go OOB
540+
weight_scales_tensor = get_param_tensor(ep, match.weight_scales_node)
541+
assert weight_scales_tensor is not None
542+
utils.align_width_and_update_state_dict(
543+
ep, match.weight_scales_node, weight_scales_tensor
544+
)
545+
546+
# Pad bias to multiple of 4 if present
547+
if match.bias_node is not None:
548+
bias_tensor = get_param_tensor(ep, match.bias_node)
549+
if bias_tensor is not None:
550+
utils.align_width_and_update_state_dict(ep, match.bias_node, bias_tensor)
551+
495552
first_graph_node = list(graph_module.graph.nodes)[0]
496553
with graph_module.graph.inserting_before(first_graph_node):
497554
weight_tensor_name = utils.get_tensor_name(ep, match.weight_node)
498555
sum_per_output_channel = weight_tensor.sum(dim=1).to(torch.int32).contiguous()
556+
557+
# Pad weight sums to align OC to multiple of 4
558+
oc = sum_per_output_channel.shape[0]
559+
if oc % 4 != 0:
560+
num_padding = 4 - (oc % 4)
561+
sum_per_output_channel = F.pad(
562+
sum_per_output_channel, (0, num_padding)
563+
).contiguous()
564+
499565
sums_name = weight_tensor_name + "_sums"
500566
sums_name = sums_name.replace(".", "_")
501567

@@ -508,7 +574,7 @@ def make_q8ta_linear_custom_op(
508574
)
509575

510576
# Use gemv variant when batch size is 1
511-
input_shape = match.fp_input_node.meta["val"].shape
577+
input_shape = match.pattern_input_node.meta["val"].shape
512578
batch_size = input_shape[-2] if len(input_shape) >= 2 else 1
513579
if batch_size == 1:
514580
op_target = exir_ops.edge.et_vk.q8ta_linear_gemv.default
@@ -520,7 +586,7 @@ def make_q8ta_linear_custom_op(
520586
"call_function",
521587
op_target,
522588
args=(
523-
match.quantize_input_node,
589+
match.pattern_input_node,
524590
match.input_scales_node,
525591
match.input_zeros_node,
526592
match.weight_node,

0 commit comments

Comments
 (0)