Skip to content

Commit 6138318

Browse files
ssjiaSS-JIA
authored andcommitted
[ET-VK][qconv] Add apply_relu support to q8ta conv operators
Pull Request resolved: pytorch#17506 The quantized convolution pattern detector correctly identifies ReLU nodes between conv output and the output quantize node, but the pattern replacement did not pass this information to the fused q8ta operator. When the pattern replaced `dequant → conv → relu → quant` with `q8ta_conv2d`, the relu node was removed from the graph but its effect was not preserved. This silently removed all conv-relu non-linearity from int8 quantized models. Add an `apply_relu` parameter throughout the full pipeline: - Custom op schemas and reference implementations (custom_ops_lib.py) - Pattern replacement (quantized_convolution.py) - C++ dispatch logic extracts apply_relu and passes it as a spec constant (Q8taConv2d.cpp, Q8taConv2dDW.cpp, Q8taConv2dPW.cpp, Q8taConv2dIm2Col.cpp) - GLSL shaders apply conditional max(value, 0) after dequantization and before requantization (q8ta_conv2d.glsl, q8ta_conv2d_dw.glsl, q8ta_conv2d_pw.glsl) - Test operator wrappers updated with proper legacy path handling (TestQ8taConv2d.cpp) ghstack-source-id: 342806070 @exported-using-ghexport Differential Revision: [D93511632](https://our.internmc.facebook.com/intern/diff/D93511632/)
1 parent 4bb7bf0 commit 6138318

15 files changed

Lines changed: 217 additions & 71 deletions

‎backends/vulkan/custom_ops_lib.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ def q8ta_conv2d(
376376
padding: list,
377377
dilation: list,
378378
groups: int,
379+
activation: str,
379380
):
380381
x = torch.ops.quantized_decomposed.dequantize_per_tensor(
381382
x, input_scale, input_zero_point, -128, 127, x.dtype
@@ -418,6 +419,9 @@ def q8ta_conv2d(
418419
x, weights, bias, stride, padding, dilation, groups
419420
)
420421

422+
if activation == "relu":
423+
out = torch.nn.functional.relu(out)
424+
421425
out = torch.ops.quantized_decomposed.quantize_per_tensor(
422426
out, output_scale, output_zero_point, -128, 127, torch.int8
423427
)
@@ -442,7 +446,8 @@ def q8ta_conv2d(
442446
SymInt[] stride,
443447
SymInt[] padding,
444448
SymInt[] dilation,
445-
SymInt groups) -> Tensor
449+
SymInt groups,
450+
str activation) -> Tensor
446451
"""
447452
)
448453
lib.impl(name, q8ta_conv2d, "CompositeExplicitAutograd")
@@ -466,7 +471,8 @@ def q8ta_conv2d(
466471
SymInt[] stride,
467472
SymInt[] padding,
468473
SymInt[] dilation,
469-
SymInt groups) -> Tensor
474+
SymInt groups,
475+
str activation) -> Tensor
470476
"""
471477
)
472478
lib.impl(name, q8ta_conv2d, "CompositeExplicitAutograd")
@@ -488,6 +494,7 @@ def q8ta_conv2d_dw(
488494
padding: list,
489495
dilation: list,
490496
groups: int,
497+
activation: str,
491498
):
492499
x = torch.ops.quantized_decomposed.dequantize_per_tensor(
493500
x, input_scale, input_zero_point, -128, 127, x.dtype
@@ -514,6 +521,9 @@ def q8ta_conv2d_dw(
514521
x, weights, bias, stride, padding, dilation, groups
515522
)
516523

524+
if activation == "relu":
525+
out = torch.nn.functional.relu(out)
526+
517527
out = torch.ops.quantized_decomposed.quantize_per_tensor(
518528
out, output_scale, output_zero_point, -128, 127, torch.int8
519529
)
@@ -538,7 +548,8 @@ def q8ta_conv2d_dw(
538548
SymInt[] stride,
539549
SymInt[] padding,
540550
SymInt[] dilation,
541-
SymInt groups) -> Tensor
551+
SymInt groups,
552+
str activation) -> Tensor
542553
"""
543554
)
544555
lib.impl(name, q8ta_conv2d_dw, "CompositeExplicitAutograd")

‎backends/vulkan/patterns/quantized_convolution.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,7 @@ def make_q8ta_conv2d_custom_op(
281281
match.padding,
282282
match.dilation,
283283
match.groups,
284+
"relu" if match.relu_node is not None else "none",
284285
),
285286
)
286287

‎backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.glsl‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ layout(push_constant) uniform restrict Block {
4747
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
4848

4949
${layout_declare_spec_const(C, "int", "apply_bias", "1")}
50+
${layout_declare_spec_const(C, "int", "activation_type", "0")}
5051

5152
// Layout specialization constants
5253
${layout_declare_spec_const(C, "int", "inp_layout", "CONTIG_LAYOUT_INT")}
@@ -220,6 +221,13 @@ void main() {
220221
}
221222
}
222223

224+
// Apply ReLU if enabled
225+
if (activation_type > 0) {
226+
[[unroll]] for (int subtile_w = 0; subtile_w < 4; ++subtile_w) {
227+
facc[subtile_w] = max(facc[subtile_w], vec4(0.0));
228+
}
229+
}
230+
223231
// Compute base output texel index (for subtile_w=0)
224232
const int base_outp_texel_idx = tensor4d_idx_to_texel_idx(outp, outp_tidx, outp_layout);
225233
const int out_w_stride = int(outp.strides[0][0]);

‎backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.glsl‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ layout(push_constant) uniform restrict Block {
4444
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
4545

4646
${layout_declare_spec_const(C, "int", "apply_bias", "1")}
47+
${layout_declare_spec_const(C, "int", "activation_type", "0")}
4748

4849
// Layout specialization constants
4950
${layout_declare_spec_const(C, "int", "inp_layout", "CONTIG_LAYOUT_INT")}
@@ -197,6 +198,13 @@ void main() {
197198
}
198199
}
199200

201+
// Apply ReLU if enabled
202+
if (activation_type > 0) {
203+
[[unroll]] for (int subtile_w = 0; subtile_w < 4; ++subtile_w) {
204+
facc[subtile_w] = max(facc[subtile_w], vec4(0.0));
205+
}
206+
}
207+
200208
// Compute base output texel index (for subtile_w=0)
201209
const int base_outp_texel_idx = tensor4d_idx_to_texel_idx(outp, outp_tidx, outp_layout);
202210
const int out_w_stride = int(outp.strides[0][0]);

‎backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.glsl‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ layout(push_constant) uniform restrict Block {
5757
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
5858

5959
${layout_declare_spec_const(C, "int", "apply_bias", "1")}
60+
${layout_declare_spec_const(C, "int", "activation_type", "0")}
6061
${layout_declare_spec_const(C, "int", "conv2d_params_K4_per_group", "1")}
6162

6263
// Layout specialization constants
@@ -197,6 +198,10 @@ void main() {
197198
fma(vec4(accum_adjusted),
198199
vec4(weight_scales[n4]) * input_scale,
199200
vec4(bias[n4]));
201+
// Apply ReLU if enabled
202+
if (activation_type > 0) {
203+
float_out_texel = max(float_out_texel, vec4(0.0));
204+
}
200205
// Requantize to int8
201206
float_out_texel =
202207
round(float_out_texel * output_inv_scale) + output_zp;
@@ -216,6 +221,10 @@ void main() {
216221
input_zp_vec * weight_sums[n4] + out_accum[m][n4];
217222
vec4 float_out_texel =
218223
vec4(accum_adjusted) * vec4(weight_scales[n4] * input_scale);
224+
// Apply ReLU if enabled
225+
if (activation_type > 0) {
226+
float_out_texel = max(float_out_texel, vec4(0.0));
227+
}
219228
// Requantize to int8
220229
float_out_texel =
221230
round(float_out_texel * output_inv_scale) + output_zp;

‎backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.cpp‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,15 @@
1717

1818
namespace vkcompute {
1919

20+
ActivationType activation_type_from_string(const std::string& activation) {
21+
if (activation == "none") {
22+
return ActivationType::kNone;
23+
} else if (activation == "relu") {
24+
return ActivationType::kRelu;
25+
}
26+
VK_THROW("Unknown activation type: ", activation);
27+
}
28+
2029
bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info) {
2130
return info.packed_dim == WHCN::kChannelsDim &&
2231
info.packed_dim_block_size == 4 &&
@@ -231,6 +240,7 @@ void add_q8ta_conv2d_node(
231240
const ValueRef padding,
232241
const ValueRef dilation,
233242
const ValueRef groups,
243+
const uint32_t activation_type,
234244
const ValueRef packed_int8_output) {
235245
(void)packed_int8_input_im2col; // Not used in general shader
236246

@@ -288,9 +298,10 @@ void add_q8ta_conv2d_node(
288298
graph.buffer_meta_ubo(packed_int8_input),
289299
graph.create_params_buffer(conv_params)};
290300

291-
// Build spec constants: apply_bias + layout constants
301+
// Build spec constants: apply_bias, apply_relu + layout constants
292302
vkapi::SpecVarList spec_constants = {
293303
apply_bias,
304+
activation_type,
294305
// Layout specialization constants
295306
graph.hashed_layout_of(packed_int8_input),
296307
graph.hashed_layout_of(packed_int8_output),
@@ -341,8 +352,12 @@ void q8ta_conv2d_general(
341352
const ValueRef padding = args.at(idx++);
342353
const ValueRef dilation = args.at(idx++);
343354
const ValueRef groups = args.at(idx++);
355+
const ValueRef activation = args.at(idx++);
344356
const ValueRef packed_int8_output = args.at(idx++);
345357

358+
uint32_t activation_type_val = static_cast<uint32_t>(
359+
activation_type_from_string(graph.extract_string(activation)));
360+
346361
QuantizationConfig weight_quant_config(8, kPerChannel, {});
347362

348363
// Prepack weight using the conv2d weight packing for the general shader
@@ -397,6 +412,7 @@ void q8ta_conv2d_general(
397412
padding,
398413
dilation,
399414
groups,
415+
activation_type_val,
400416
packed_int8_output);
401417
}
402418

‎backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,13 @@
1313

1414
namespace vkcompute {
1515

16+
enum class ActivationType : uint32_t {
17+
kNone = 0,
18+
kRelu = 1,
19+
};
20+
21+
ActivationType activation_type_from_string(const std::string& activation);
22+
1623
bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info);
1724

1825
bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info);
@@ -58,6 +65,7 @@ void add_q8ta_conv2d_dw_node(
5865
const ValueRef padding,
5966
const ValueRef dilation,
6067
const ValueRef groups,
68+
const uint32_t activation_type,
6169
const ValueRef packed_int8_output);
6270

6371
void add_conv2d_dw_q8ta_q8csw_q8to_4w4c_node(
@@ -97,6 +105,7 @@ void add_q8ta_conv2d_node(
97105
const ValueRef padding,
98106
const ValueRef dilation,
99107
const ValueRef groups,
108+
const uint32_t activation_type,
100109
const ValueRef packed_int8_output);
101110

102111
void add_q8ta_conv2d_pw_node(
@@ -111,6 +120,7 @@ void add_q8ta_conv2d_pw_node(
111120
const ValueRef output_zp,
112121
const ValueRef bias_data,
113122
const ValueRef packed_bias,
123+
const uint32_t activation_type,
114124
const ValueRef packed_int8_output);
115125

116126
void q8ta_conv2d_im2col(ComputeGraph& graph, const std::vector<ValueRef>& args);

‎backends/vulkan/runtime/graph/ops/impl/Q8taConv2dDW.cpp‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,7 @@ void add_q8ta_conv2d_dw_node(
281281
const ValueRef padding,
282282
const ValueRef dilation,
283283
const ValueRef groups,
284+
const uint32_t activation_type,
284285
const ValueRef packed_int8_output) {
285286
Conv2DParams conv_params = create_conv2d_params(
286287
graph,
@@ -334,9 +335,10 @@ void add_q8ta_conv2d_dw_node(
334335
graph.buffer_meta_ubo(packed_int8_input),
335336
graph.create_params_buffer(conv_params)};
336337

337-
// Build spec constants: apply_bias + layout constants
338+
// Build spec constants: apply_bias, activation_type + layout constants
338339
vkapi::SpecVarList spec_constants = {
339340
apply_bias,
341+
activation_type,
340342
// Layout specialization constants
341343
graph.hashed_layout_of(packed_int8_input),
342344
graph.hashed_layout_of(packed_int8_output),
@@ -385,8 +387,12 @@ void q8ta_conv2d_dw(ComputeGraph& graph, const std::vector<ValueRef>& args) {
385387
const ValueRef padding = args.at(idx++);
386388
const ValueRef dilation = args.at(idx++);
387389
const ValueRef groups = args.at(idx++);
390+
const ValueRef activation = args.at(idx++);
388391
const ValueRef packed_int8_output = args.at(idx++);
389392

393+
uint32_t activation_type_val = static_cast<uint32_t>(
394+
activation_type_from_string(graph.extract_string(activation)));
395+
390396
QuantizationConfig weight_quant_config(8, kPerChannel, {});
391397

392398
// Prepack weight using depthwise-specific packing
@@ -432,6 +438,7 @@ void q8ta_conv2d_dw(ComputeGraph& graph, const std::vector<ValueRef>& args) {
432438
padding,
433439
dilation,
434440
groups,
441+
activation_type_val,
435442
packed_int8_output);
436443
}
437444

‎backends/vulkan/runtime/graph/ops/impl/Q8taConv2dIm2Col.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ void q8ta_conv2d_im2col(
197197
const ValueRef padding = args.at(idx++);
198198
const ValueRef dilation = args.at(idx++);
199199
const ValueRef groups = args.at(idx++);
200+
const ValueRef activation = args.at(idx++);
200201
const ValueRef packed_int8_output = args.at(idx++);
201202

202203
QuantizationConfig weight_quant_config(8, kPerChannel, {});
@@ -225,6 +226,9 @@ void q8ta_conv2d_im2col(
225226
prepack_standard(graph, bias_data, utils::kBuffer, utils::kWidthPacked);
226227
}
227228

229+
uint32_t activation_type_val = static_cast<uint32_t>(
230+
activation_type_from_string(graph.extract_string(activation)));
231+
228232
// Calculate im2col output sizes
229233
std::vector<int64_t> im2col_sizes = calculate_q8ta_im2col_sizes(
230234
&graph, packed_int8_input, packed_int8_output, kernel_size, groups);
@@ -265,6 +269,7 @@ void q8ta_conv2d_im2col(
265269
output_zp,
266270
bias_data,
267271
packed_bias,
272+
activation_type_val,
268273
packed_int8_output);
269274
}
270275

‎backends/vulkan/runtime/graph/ops/impl/Q8taConv2dPW.cpp‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ void add_q8ta_conv2d_pw_node(
199199
const ValueRef output_zp,
200200
const ValueRef bias_data,
201201
const ValueRef packed_bias,
202+
const uint32_t activation_type,
202203
const ValueRef packed_int8_output) {
203204
// Validate packed dim info for input and output tensors
204205
// To maximize performance, the input tensor must be in 4W4C layout
@@ -242,9 +243,10 @@ void add_q8ta_conv2d_pw_node(
242243
graph.buffer_meta_ubo(packed_int8_output),
243244
graph.buffer_meta_ubo(packed_int8_input)};
244245

245-
// Build spec constants: apply_bias + layout constants
246+
// Build spec constants: apply_bias, activation_type + layout constants
246247
vkapi::SpecVarList spec_constants = {
247248
apply_bias,
249+
activation_type,
248250
K4_per_group,
249251
// Layout specialization constants
250252
graph.hashed_layout_of(packed_int8_output),
@@ -296,8 +298,12 @@ void q8ta_conv2d_pw(ComputeGraph& graph, const std::vector<ValueRef>& args) {
296298
(void)args.at(idx++); // padding
297299
(void)args.at(idx++); // dilation
298300
(void)args.at(idx++); // groups
301+
const ValueRef activation_ref = args.at(idx++);
299302
const ValueRef packed_int8_output = args.at(idx++);
300303

304+
uint32_t activation_type_val = static_cast<uint32_t>(
305+
activation_type_from_string(graph.extract_string(activation_ref)));
306+
301307
QuantizationConfig weight_quant_config(8, kPerChannel, {});
302308

303309
// Prepack weight using pointwise-specific packing
@@ -342,6 +348,7 @@ void q8ta_conv2d_pw(ComputeGraph& graph, const std::vector<ValueRef>& args) {
342348
output_zp,
343349
bias_data,
344350
packed_bias,
351+
activation_type_val,
345352
packed_int8_output);
346353
}
347354

0 commit comments

Comments
 (0)