Skip to content

Commit 187dc1d

Browse files
ssjiaSS-JIA
authored andcommitted
[ET-VK][q8ta] Add q8ta_linear_gemv op for batch-1 int8 linear
Pull Request resolved: pytorch#17566 Add a cooperative GEMV variant of q8ta_linear optimized for batch size 1. The existing q8ta_linear uses a tiled algorithm with 4H4W packed int8 layout, which is inefficient for single-row inputs because it wastes 3/4 of each ivec4 block. The new q8ta_linear_gemv uses 4W packed int8 layout (scalar int[] buffers) and a cooperative algorithm where 64 threads split the K reduction dimension with shared memory tree reduction. The shader loads one packed int32 (4 int8 values) per thread per K iteration and accumulates dot products against the weight tile using dotPacked4x8AccSatEXT. After reduction, thread 0 applies scales, zero points, bias, and quantizes the output. The pattern matcher in quantized_linear.py selects q8ta_linear_gemv when the input batch dimension is 1, falling back to q8ta_linear for larger batches. Also adds PACKED_INT8_4W (value 5) to the serialization schema to support the 4W memory layout in the export pipeline. Authored with Claude. ghstack-source-id: 343460519 @exported-using-ghexport Differential Revision: [D93768643](https://our.internmc.facebook.com/intern/diff/D93768643/)
1 parent 3087563 commit 187dc1d

10 files changed

Lines changed: 608 additions & 16 deletions

File tree

‎backends/vulkan/custom_ops_lib.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,71 @@ def q8ta_linear(
421421
lib.impl(name, q8ta_linear, "CompositeExplicitAutograd")
422422
q8ta_linear_op = getattr(getattr(torch.ops, namespace), name)
423423

424+
#######################
425+
## q8ta_linear_gemv ##
426+
#######################
427+
428+
429+
def q8ta_linear_gemv(
430+
x: torch.Tensor,
431+
input_scale: float,
432+
input_zero_point: int,
433+
weights: torch.Tensor,
434+
weight_sums: torch.Tensor,
435+
weight_scales: torch.Tensor,
436+
output_scale: float,
437+
output_zero_point: int,
438+
bias: Optional[torch.Tensor] = None,
439+
activation: str = "none",
440+
):
441+
weight_zeros = torch.zeros_like(weight_scales, dtype=torch.int32)
442+
weights = torch.ops.quantized_decomposed.dequantize_per_channel(
443+
weights,
444+
weight_scales,
445+
weight_zeros,
446+
0,
447+
-127,
448+
127,
449+
torch.int8,
450+
)
451+
452+
x = torch.ops.quantized_decomposed.dequantize_per_tensor(
453+
x, input_scale, input_zero_point, -128, 127, x.dtype
454+
)
455+
456+
out = torch.nn.functional.linear(x, weights)
457+
if bias is not None:
458+
out = out + bias
459+
460+
if activation == "relu":
461+
out = torch.nn.functional.relu(out)
462+
463+
out = torch.ops.quantized_decomposed.quantize_per_tensor(
464+
out, output_scale, output_zero_point, -128, 127, torch.int8
465+
)
466+
467+
return out
468+
469+
470+
name = "q8ta_linear_gemv"
471+
lib.define(
472+
f"""
473+
{name}(
474+
Tensor x,
475+
float input_scale,
476+
int input_zero_point,
477+
Tensor weights,
478+
Tensor weight_sums,
479+
Tensor weight_scales,
480+
float output_scale,
481+
int output_zero_point,
482+
Tensor? bias = None,
483+
str activation = "none") -> Tensor
484+
"""
485+
)
486+
lib.impl(name, q8ta_linear_gemv, "CompositeExplicitAutograd")
487+
q8ta_linear_gemv_op = getattr(getattr(torch.ops, namespace), name)
488+
424489
###################
425490
## q8ta_conv2d_* ##
426491
###################

‎backends/vulkan/op_registry.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -858,6 +858,29 @@ def register_q8ta_linear():
858858
)
859859

860860

861+
@update_features(exir_ops.edge.et_vk.q8ta_linear_gemv.default)
862+
def register_q8ta_linear_gemv():
863+
return OpFeatures(
864+
inputs_storage=[
865+
utils.PACKED_INT8_4W_BUFFER, # input
866+
utils.NO_STORAGE, # input_scale (non tensor)
867+
utils.NO_STORAGE, # input_zero_point (non tensor)
868+
utils.NO_STORAGE, # weight (prepacked)
869+
utils.NO_STORAGE, # weight_sums (prepacked)
870+
utils.NO_STORAGE, # weight_scales (prepacked)
871+
utils.NO_STORAGE, # output_scale (non tensor)
872+
utils.NO_STORAGE, # output_zero_point (non tensor)
873+
utils.NO_STORAGE, # bias (prepacked)
874+
utils.NO_STORAGE, # activation (non tensor)
875+
],
876+
outputs_storage=[
877+
utils.PACKED_INT8_4W_BUFFER,
878+
],
879+
supports_resize=False,
880+
supports_prepacking=True,
881+
)
882+
883+
861884
# =============================================================================
862885
# SDPA.cpp
863886
# =============================================================================

‎backends/vulkan/patterns/quantized_linear.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -507,10 +507,18 @@ def make_q8ta_linear_custom_op(
507507
data=sum_per_output_channel,
508508
)
509509

510+
# Use gemv variant when batch size is 1
511+
input_shape = match.fp_input_node.meta["val"].shape
512+
batch_size = input_shape[-2] if len(input_shape) >= 2 else 1
513+
if batch_size == 1:
514+
op_target = exir_ops.edge.et_vk.q8ta_linear_gemv.default
515+
else:
516+
op_target = exir_ops.edge.et_vk.q8ta_linear.default
517+
510518
with graph_module.graph.inserting_before(match.output_node):
511519
qlinear_node = graph_module.graph.create_node(
512520
"call_function",
513-
exir_ops.edge.et_vk.q8ta_linear.default,
521+
op_target,
514522
args=(
515523
match.quantize_input_node,
516524
match.input_scales_node,
Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#version 450 core
10+
11+
${define_required_extensions("buffer", DTYPE)}
12+
13+
#extension GL_EXT_control_flow_attributes : require
14+
#extension GL_EXT_integer_dot_product : require
15+
16+
#define PRECISION ${PRECISION}
17+
#define VEC4_T ${texel_load_type(DTYPE, "buffer")}
18+
#define T int
19+
20+
$if WEIGHT_STORAGE == "buffer":
21+
#define WEIGHT_BUFFER
22+
23+
#define TILE_K4 ${TILE_K4}
24+
#define TILE_N4 ${TILE_N4}
25+
26+
#define TILE_M4 1
27+
#define TILE_M 1
28+
#define TILE_K ${TILE_K4 * 4}
29+
#define TILE_N ${TILE_N4 * 4}
30+
31+
#define WGS ${WGS}
32+
33+
layout(std430) buffer;
34+
35+
// Scalar int arrays for 4W packed int8 input/output
36+
${layout_declare_tensor(B, "w", "t_packed_int8_output", "int", "buffer")}
37+
${layout_declare_tensor(B, "r", "t_packed_int8_input", "int", "buffer")}
38+
// Weight uses ivec4 (same format as q8ta_linear)
39+
${layout_declare_tensor(B, "r", "t_packed_int8_weight", "int", WEIGHT_STORAGE, is_scalar_array=False)}
40+
${layout_declare_tensor(B, "r", "t_weight_sums", "int", "buffer", is_scalar_array=False)}
41+
${layout_declare_tensor(B, "r", "t_weight_scales", DTYPE, "buffer", is_scalar_array=False)}
42+
${layout_declare_tensor(B, "r", "t_bias", DTYPE, "buffer", is_scalar_array=False)}
43+
44+
${layout_declare_spec_const(C, "int", "apply_bias", "0")}
45+
${layout_declare_spec_const(C, "int", "activation_type", "0")}
46+
47+
${layout_declare_ubo(B, "ivec4", "output_sizes")}
48+
${layout_declare_ubo(B, "ivec4", "input_sizes")}
49+
50+
layout(push_constant) uniform restrict Block {
51+
float input_scale;
52+
int input_zp;
53+
float output_inv_scale;
54+
int output_zp;
55+
};
56+
57+
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
58+
59+
#include "common.glslh"
60+
#include "linear_int8_weight_tile_load.glslh"
61+
#include "linear_fp_output_tile_int8_int8_compute.glslh"
62+
#include "linear_fp_weight_scales_load.glslh"
63+
#include "linear_int_weight_sums_load.glslh"
64+
#include "linear_fp_bias_load.glslh"
65+
66+
shared Int32Accum partial_accums[WGS];
67+
68+
void main() {
69+
const int lid = int(gl_LocalInvocationID.z);
70+
const int n4 = int(gl_GlobalInvocationID.x) * TILE_N4;
71+
72+
const int n = mul_4(n4);
73+
74+
const int K4 = div_up_4(input_sizes.x);
75+
const int N4 = div_up_4(output_sizes.x);
76+
77+
if (n >= output_sizes.x) {
78+
return;
79+
}
80+
81+
Int32Accum out_accum;
82+
initialize(out_accum);
83+
84+
Int8WeightTile int8_weight_tile;
85+
86+
for (int k4 = lid; k4 < K4; k4 += WGS) {
87+
// Load one packed int32 from the 4W input buffer. Each int32 contains
88+
// 4 int8 values at k=k4*4..k4*4+3.
89+
const int packed_input = t_packed_int8_input[k4];
90+
91+
load_int8_weight_tile(int8_weight_tile, n4, k4, N4);
92+
93+
// Accumulate dot products of the input int8x4 with each weight int8x4
94+
[[unroll]] for (int n = 0; n < TILE_N; ++n) {
95+
const int tile_n4 = div_4(n);
96+
const int n4i = mod_4(n);
97+
out_accum.data[0][tile_n4][n4i] = dotPacked4x8AccSatEXT(
98+
packed_input,
99+
int8_weight_tile.data[0][tile_n4][n4i],
100+
out_accum.data[0][tile_n4][n4i]);
101+
}
102+
}
103+
104+
partial_accums[lid] = out_accum;
105+
106+
memoryBarrierShared();
107+
barrier();
108+
109+
// Only the first thread writes the result
110+
if (lid == 0) {
111+
for (int i = 1; i < WGS; ++i) {
112+
[[unroll]] for (int tile_n4 = 0; tile_n4 < TILE_N4; ++tile_n4) {
113+
out_accum.data[0][tile_n4] +=
114+
partial_accums[i].data[0][tile_n4];
115+
}
116+
}
117+
118+
FPPerOutChannelParams weight_scales_tile;
119+
load_weight_scales_tile(weight_scales_tile, n4);
120+
121+
IntPerOutChannelParams weight_sums_tile;
122+
load_weight_sums_tile(weight_sums_tile, n4);
123+
124+
FPOutTile out_tile;
125+
initialize(out_tile);
126+
127+
if (apply_bias > 0) {
128+
FPPerOutChannelParams bias_tile;
129+
load_bias_tile(bias_tile, n4);
130+
131+
accumulate_out_tile_with_int_accum(
132+
out_tile,
133+
out_accum,
134+
input_scale,
135+
input_zp,
136+
weight_sums_tile,
137+
weight_scales_tile,
138+
bias_tile);
139+
} else {
140+
accumulate_out_tile_with_int_accum(
141+
out_tile,
142+
out_accum,
143+
input_scale,
144+
input_zp,
145+
weight_sums_tile,
146+
weight_scales_tile);
147+
}
148+
149+
// Apply ReLU if enabled
150+
if (activation_type > 0) {
151+
[[unroll]] for (int tile_n4 = 0; tile_n4 < TILE_N4; ++tile_n4) {
152+
out_tile.data[0][tile_n4] = max(out_tile.data[0][tile_n4], vec4(0.0));
153+
}
154+
}
155+
156+
// Quantize and write to scalar int[] buffer. Each int32 at position n4
157+
// contains 4 packed int8 output values for channels n4*4..n4*4+3.
158+
[[unroll]] for (int tile_n4 = 0; tile_n4 < TILE_N4; ++tile_n4) {
159+
if (n4 + tile_n4 < N4) {
160+
t_packed_int8_output[n4 + tile_n4] = quantize_and_pack(
161+
out_tile.data[0][tile_n4], output_inv_scale, output_zp);
162+
}
163+
}
164+
}
165+
}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
q8ta_linear_gemv:
8+
parameter_names_with_default_values:
9+
DTYPE: float
10+
WEIGHT_STORAGE: texture2d
11+
TILE_K4: 1
12+
TILE_N4: 2
13+
WGS: 64
14+
generate_variant_forall:
15+
DTYPE:
16+
- VALUE: float
17+
shader_variants:
18+
- NAME: q8ta_linear_gemv

0 commit comments

Comments
 (0)