Skip to content

Commit 73ae813

Browse files
authored
Vulkan Q8 Conv2D: specialize shader on static parameters and tensor sizes (pytorch#16036)
This change moves all fixed Conv2D parameters (kernel shape, stride, padding, dilation, groups) and the input/output tensor dimensions into Vulkan specialization constants. By making these values compile-time constants, the backend can generate more optimized pipelines, eliminate generic fallback paths, and reduce dynamic indexing overhead. This significantly improves performance across large and compute-intensive convolution workloads. cc @SS-JIA @manuelcandales @digantdesai @cbilgin Signed-off-by: Pablo Marquez Tello <pablo.tello@arm.com>
1 parent 0ee2f49 commit 73ae813

16 files changed

Lines changed: 318 additions & 99 deletions

‎backends/vulkan/runtime/graph/ops/glsl/col2im.glsl‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,23 @@ ${layout_declare_ubo(B, "ivec4", "input_sizes")}
4141
// Sizes of the im2col matrix of the convolution output
4242
${layout_declare_ubo(B, "ivec4", "matrix_sizes")}
4343

44-
${layout_declare_ubo(B, "Conv2DParams", "conv2d_params")}
44+
${layout_declare_spec_const(C, "int", "apply_bias", "1")}
45+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_x", "1")}
46+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_y", "1")}
47+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_x", "1")}
48+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_y", "1")}
49+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_x", "1")}
50+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_y", "1")}
51+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_x", "1")}
52+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_y", "1")}
53+
${layout_declare_spec_const(C, "int", "conv2d_params_in_channels_per_group", "1")}
54+
${layout_declare_spec_const(C, "int", "conv2d_params_out_channels_per_group", "1")}
55+
${layout_declare_spec_const(C, "int", "conv2d_params_K4_per_group", "1")}
56+
${layout_declare_spec_const(C, "int", "conv2d_params_K4", "1")}
57+
${layout_declare_spec_const(C, "int", "conv2d_params_K_per_group", "1")}
58+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K", "1")}
59+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")}
60+
${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")}
4561

4662
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
4763

‎backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.glsl‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ ${layout_declare_tensor(B, "r", "t_bias", DTYPE, "buffer", is_scalar_array=False
3636

3737
${layout_declare_ubo(B, "ivec4", "output_sizes")}
3838
${layout_declare_ubo(B, "ivec4", "input_sizes")}
39-
${layout_declare_ubo(B, "Conv2DParams", "conv2d_params")}
4039

4140
layout(push_constant) uniform restrict Block {
4241
float input_scale;
@@ -48,6 +47,24 @@ layout(push_constant) uniform restrict Block {
4847
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
4948

5049
${layout_declare_spec_const(C, "int", "apply_bias", "1")}
50+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_x", "1")}
51+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_y", "1")}
52+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_x", "1")}
53+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_y", "1")}
54+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_x", "1")}
55+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_y", "1")}
56+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_x", "1")}
57+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_y", "1")}
58+
${layout_declare_spec_const(C, "int", "conv2d_params_in_channels_per_group", "1")}
59+
${layout_declare_spec_const(C, "int", "conv2d_params_out_channels_per_group", "1")}
60+
${layout_declare_spec_const(C, "int", "conv2d_params_K4_per_group", "1")}
61+
${layout_declare_spec_const(C, "int", "conv2d_params_K4", "1")}
62+
${layout_declare_spec_const(C, "int", "conv2d_params_K_per_group", "1")}
63+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K", "1")}
64+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")}
65+
${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")}
66+
67+
5168

5269
#include "conv2d_dw_q8_utils.glslh"
5370

@@ -67,21 +84,21 @@ void main() {
6784

6885
Conv2dBlockExtents in_block_extents = make_block_extents(input_sizes);
6986

70-
const int Kw4 = div_up_4(conv2d_params.kernel_size.x);
87+
const int Kw4 = div_up_4(conv2d_params_kernel_size_x);
7188

7289
// Compute 4 channels for 4 output elements.
7390
ivec4 acc[4];
7491
[[unroll]] for (int i = 0; i < 4; ++i) {
7592
acc[i] = ivec4(0);
7693
}
7794

78-
for (int ky = 0; ky < conv2d_params.kernel_size.y; ky++) {
79-
const int h = out_h * conv2d_params.stride.y - conv2d_params.padding.y +
80-
ky * conv2d_params.dilation.y;
95+
for (int ky = 0; ky < conv2d_params_kernel_size_y; ky++) {
96+
const int h = out_h * conv2d_params_stride_y - conv2d_params_padding_y +
97+
ky * conv2d_params_dilation_y;
8198

82-
for (int kx = 0; kx < conv2d_params.kernel_size.x; kx++) {
83-
const int w = out_w * conv2d_params.stride.x - conv2d_params.padding.x +
84-
kx * conv2d_params.dilation.x;
99+
for (int kx = 0; kx < conv2d_params_kernel_size_x; kx++) {
100+
const int w = out_w * conv2d_params_stride_x - conv2d_params_padding_x +
101+
kx * conv2d_params_dilation_x;
85102

86103
// Load and unpack weights.
87104
const int packed_weight_4c = load_weight_1w4c(
@@ -96,7 +113,7 @@ void main() {
96113

97114
[[unroll]] for (int subtile_w = 0; subtile_w < 4; ++subtile_w) {
98115
ivec4 input_texel = unpack_int8x4(load_input_1w4c(
99-
w + conv2d_params.stride.x * subtile_w,
116+
w + conv2d_params_stride_x * subtile_w,
100117
h,
101118
out_block_idx.data.z,
102119
out_block_extents.data.z,

‎backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block.glslh‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -63,27 +63,27 @@ void im2col_idx_to_input_tidx(
6363
TensorIndex4D output_tidx;
6464
unwrap_m(output_tidx, im2col_idx.row);
6565

66-
const int in_channels_per_group = conv2d_params.in_channels_per_group;
66+
const int in_channels_per_group = conv2d_params_in_channels_per_group;
6767
// Determine the corresponding position within the convolution window based
6868
// on the col index (more specifically, the col index within the group)
6969
const int channel_within_group =
7070
im2col_idx.col_idx_in_group % in_channels_per_group;
7171
const int kernel_x = (im2col_idx.col_idx_in_group / in_channels_per_group) %
72-
conv2d_params.kernel_size.x;
72+
conv2d_params_kernel_size_x;
7373
const int kernel_y = im2col_idx.col_idx_in_group /
74-
(in_channels_per_group * conv2d_params.kernel_size.x);
74+
(in_channels_per_group * conv2d_params_kernel_size_x);
7575

7676
// Calculate the actual input channel index
7777
const int channel_idx =
78-
im2col_idx.group_idx * conv2d_params.in_channels_per_group +
78+
im2col_idx.group_idx * conv2d_params_in_channels_per_group +
7979
channel_within_group;
8080

8181
// Calculate corresponding input coordinates based on output position
8282
// associated with the row index.
83-
const int input_y = int(output_tidx.data.y * conv2d_params.stride.y) -
84-
int(conv2d_params.padding.y) + int(kernel_y * conv2d_params.dilation.y);
85-
const int input_x = int(output_tidx.data.x * conv2d_params.stride.x) -
86-
int(conv2d_params.padding.x) + int(kernel_x * conv2d_params.dilation.x);
83+
const int input_y = int(output_tidx.data.y * conv2d_params_stride_y) -
84+
int(conv2d_params_padding_y) + int(kernel_y * conv2d_params_dilation_y);
85+
const int input_x = int(output_tidx.data.x * conv2d_params_stride_x) -
86+
int(conv2d_params_padding_x) + int(kernel_x * conv2d_params_dilation_x);
8787

8888
input_tidx.data = ivec4(input_x, input_y, channel_idx, output_tidx.data.w);
8989
}

‎backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_load.glslh‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ void load_im2col_block_fast(
6464
// Due to the assumption that in_channels_per_group % 4 == 0, it is
6565
// guaranteed that the next 4 columns (including this one) is part of the
6666
// same group.
67-
im2col_idx.group_idx = im2col_idx.col / conv2d_params.K_per_group;
68-
im2col_idx.col_idx_in_group = im2col_idx.col % conv2d_params.K_per_group;
67+
im2col_idx.group_idx = im2col_idx.col / conv2d_params_K_per_group;
68+
im2col_idx.col_idx_in_group = im2col_idx.col % conv2d_params_K_per_group;
6969

7070
[[unroll]] for (int m_off = 0; m_off < 4; ++m_off) {
7171
if (im2col_idx.row >= M) {
@@ -98,9 +98,9 @@ void load_im2col_block_slow(
9898
im2col_idx_base.col = mul_4(k4);
9999
im2col_idx_base.row = mul_4(m4);
100100

101-
im2col_idx_base.group_idx = im2col_idx_base.col / conv2d_params.K_per_group;
101+
im2col_idx_base.group_idx = im2col_idx_base.col / conv2d_params_K_per_group;
102102
im2col_idx_base.col_idx_in_group =
103-
im2col_idx_base.col % conv2d_params.K_per_group;
103+
im2col_idx_base.col % conv2d_params_K_per_group;
104104

105105
[[unroll]] for (int m_off = 0; m_off < 4; ++m_off) {
106106
[[unroll]] for (int k_off = 0; k_off < 4; ++k_off) {
@@ -109,7 +109,7 @@ void load_im2col_block_slow(
109109
im2col_idx.col_idx_in_group += k_off;
110110

111111
// bounds checking
112-
if (im2col_idx.col_idx_in_group >= conv2d_params.logical_K_per_group ||
112+
if (im2col_idx.col_idx_in_group >= conv2d_params_logical_K_per_group ||
113113
im2col_idx.row >= M) {
114114
block.data[m_off][k_off] = T(0);
115115
continue;
@@ -129,7 +129,7 @@ void load_im2col_block(
129129
const int m4,
130130
const int logical_K,
131131
const int M) {
132-
if (mod_4(conv2d_params.in_channels_per_group) == 0) {
132+
if (mod_4(conv2d_params_in_channels_per_group) == 0) {
133133
load_im2col_block_fast(block, k4, m4, logical_K, M);
134134
} else {
135135
load_im2col_block_slow(block, k4, m4, logical_K, M);

‎backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ ${layout_declare_tensor(B, "r", "t_bias", DTYPE, "buffer", is_scalar_array=False
4444

4545
${layout_declare_ubo(B, "ivec4", "output_sizes")}
4646
${layout_declare_ubo(B, "ivec4", "input_sizes")}
47-
${layout_declare_ubo(B, "Conv2DParams", "conv2d_params")}
4847

4948
layout(push_constant) uniform restrict Block {
5049
float input_scale;
@@ -56,6 +55,23 @@ layout(push_constant) uniform restrict Block {
5655
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
5756

5857
${layout_declare_spec_const(C, "int", "apply_bias", "1")}
58+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_x", "1")}
59+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_y", "1")}
60+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_x", "1")}
61+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_y", "1")}
62+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_x", "1")}
63+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_y", "1")}
64+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_x", "1")}
65+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_y", "1")}
66+
${layout_declare_spec_const(C, "int", "conv2d_params_in_channels_per_group", "1")}
67+
${layout_declare_spec_const(C, "int", "conv2d_params_out_channels_per_group", "1")}
68+
${layout_declare_spec_const(C, "int", "conv2d_params_K4_per_group", "1")}
69+
${layout_declare_spec_const(C, "int", "conv2d_params_K4", "1")}
70+
${layout_declare_spec_const(C, "int", "conv2d_params_K_per_group", "1")}
71+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K", "1")}
72+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")}
73+
${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")}
74+
5975

6076
#include "conv2d_int8_input_tile_load.glslh"
6177
#include "linear_int8_weight_tile_load.glslh"
@@ -88,7 +104,7 @@ void main() {
88104
Int8InputTileIndex input_idx = make_initial_int8_input_tile_index(
89105
output_block_idx, input_block_extents);
90106

91-
for (int k4 = 0; k4 < conv2d_params.K4_per_group; k4++) {
107+
for (int k4 = 0; k4 < conv2d_params_K4_per_group; k4++) {
92108
load_packed_int8_input_tile(int8_input_tile, input_idx);
93109

94110
load_int8_weight_tile(

‎backends/vulkan/runtime/graph/ops/glsl/conv2d_q8_utils.glslh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ void perform_conv1d(
106106
const ivec4 weight_block,
107107
const int kx) {
108108
[[unroll]] for (int out_w = 0; out_w < 4; ++out_w) {
109-
const int window_i = out_w * conv2d_params.stride.x + kx;
109+
const int window_i = out_w * conv2d_params_stride_x + kx;
110110
[[unroll]] for (int out_c = 0; out_c < 4; ++out_c) {
111111
accum.data[out_w][0][out_c] = dotPacked4x8AccSatEXT(
112112
input_window.data[window_i],

‎backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.glsl‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,27 @@ ${layout_declare_tensor(B, "r", "t_bias", DTYPE, "buffer", is_scalar_array=False
4141

4242
${layout_declare_ubo(B, "ivec4", "output_sizes")}
4343
${layout_declare_ubo(B, "ivec4", "input_sizes")}
44-
${layout_declare_ubo(B, "Conv2DParams", "conv2d_params")}
4544

4645
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
4746

4847
${layout_declare_spec_const(C, "int", "apply_bias", "1")}
48+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_x", "1")}
49+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_y", "1")}
50+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_x", "1")}
51+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_y", "1")}
52+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_x", "1")}
53+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_y", "1")}
54+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_x", "1")}
55+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_y", "1")}
56+
${layout_declare_spec_const(C, "int", "conv2d_params_in_channels_per_group", "1")}
57+
${layout_declare_spec_const(C, "int", "conv2d_params_out_channels_per_group", "1")}
58+
${layout_declare_spec_const(C, "int", "conv2d_params_K4_per_group", "1")}
59+
${layout_declare_spec_const(C, "int", "conv2d_params_K4", "1")}
60+
${layout_declare_spec_const(C, "int", "conv2d_params_K_per_group", "1")}
61+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K", "1")}
62+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")}
63+
${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")}
64+
4965

5066
#include "linear_fp_input_tile_load.glslh"
5167
#include "linear_int8_weight_tile_load.glslh"
@@ -75,10 +91,10 @@ void main() {
7591
return;
7692
}
7793

78-
const int group_idx = n / conv2d_params.out_channels_per_group;
79-
const int input_k4_offset = conv2d_params.K4_per_group * group_idx;
94+
const int group_idx = n / conv2d_params_out_channels_per_group;
95+
const int input_k4_offset = conv2d_params_K4_per_group * group_idx;
8096

81-
const int K4 = conv2d_params.K4;
97+
const int K4 = conv2d_params_K4;
8298
const int N4 = div_up_4(N);
8399

84100
FPOutTile out_tile;
@@ -90,13 +106,13 @@ void main() {
90106
const bool dont_check_bounds = (M - m) >= TILE_M;
91107

92108
if (dont_check_bounds) {
93-
for (int k4 = 0; k4 < conv2d_params.K4_per_group; k4++) {
109+
for (int k4 = 0; k4 < conv2d_params_K4_per_group; k4++) {
94110
load_input_tile_no_checks(in_tile, k4 + input_k4_offset, m, K4, M);
95111
load_int8_weight_tile(int8_weight_tile, n4, k4, N4);
96112
fp_accumulate_with_int8_weight(out_tile, in_tile, int8_weight_tile);
97113
}
98114
} else {
99-
for (int k4 = 0; k4 < conv2d_params.K4_per_group; k4++) {
115+
for (int k4 = 0; k4 < conv2d_params_K4_per_group; k4++) {
100116
load_input_tile_with_checks(in_tile, k4 + input_k4_offset, m, K4, M);
101117
load_int8_weight_tile(int8_weight_tile, n4, k4, N4);
102118
fp_accumulate_with_int8_weight(out_tile, in_tile, int8_weight_tile);

‎backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.glsl‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ ${layout_declare_tensor(B, "r", "t_bias", DTYPE, "buffer", is_scalar_array=False
4444

4545
${layout_declare_ubo(B, "ivec4", "output_sizes")}
4646
${layout_declare_ubo(B, "ivec4", "input_sizes")}
47-
${layout_declare_ubo(B, "Conv2DParams", "conv2d_params")}
4847

4948
layout(push_constant) uniform restrict Block {
5049
float input_scale;
@@ -54,6 +53,23 @@ layout(push_constant) uniform restrict Block {
5453
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
5554

5655
${layout_declare_spec_const(C, "int", "apply_bias", "1")}
56+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_x", "1")}
57+
${layout_declare_spec_const(C, "int", "conv2d_params_stride_y", "1")}
58+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_x", "1")}
59+
${layout_declare_spec_const(C, "int", "conv2d_params_padding_y", "1")}
60+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_x", "1")}
61+
${layout_declare_spec_const(C, "int", "conv2d_params_dilation_y", "1")}
62+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_x", "1")}
63+
${layout_declare_spec_const(C, "int", "conv2d_params_kernel_size_y", "1")}
64+
${layout_declare_spec_const(C, "int", "conv2d_params_in_channels_per_group", "1")}
65+
${layout_declare_spec_const(C, "int", "conv2d_params_out_channels_per_group", "1")}
66+
${layout_declare_spec_const(C, "int", "conv2d_params_K4_per_group", "1")}
67+
${layout_declare_spec_const(C, "int", "conv2d_params_K4", "1")}
68+
${layout_declare_spec_const(C, "int", "conv2d_params_K_per_group", "1")}
69+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K", "1")}
70+
${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")}
71+
${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")}
72+
5773

5874
#include "linear_int8_input_tile_load.glslh"
5975
#include "linear_int8_weight_tile_load.glslh"
@@ -83,10 +99,10 @@ void main() {
8399
return;
84100
}
85101

86-
const int group_idx = n / conv2d_params.out_channels_per_group;
87-
const int input_k4_offset = conv2d_params.K4_per_group * group_idx;
102+
const int group_idx = n / conv2d_params_out_channels_per_group;
103+
const int input_k4_offset = conv2d_params_K4_per_group * group_idx;
88104

89-
const int K4 = conv2d_params.K4;
105+
const int K4 = conv2d_params_K4;
90106
const int N4 = div_up_4(N);
91107

92108
Int32Accum out_accum;
@@ -95,7 +111,7 @@ void main() {
95111
Int8InputTile int8_in_tile;
96112
Int8WeightTile int8_weight_tile;
97113

98-
for (int k4 = 0; k4 < conv2d_params.K4_per_group; k4++) {
114+
for (int k4 = 0; k4 < conv2d_params_K4_per_group; k4++) {
99115
load_int8_input_tile(int8_in_tile, k4 + input_k4_offset, m4, K4);
100116
load_int8_weight_tile(int8_weight_tile, n4, k4, N4);
101117

0 commit comments

Comments
 (0)