Skip to content

Commit 6f424e5

Browse files
ssjiaSS-JIA
authored andcommitted
[ET-VK] Modernize concat ops with indexing.glslh and safe UBO indexing
Pull Request resolved: pytorch#18453 Modernize the concat operator implementation to use current best practices: - Rewrite concat_buffer.glsl and concat_texture.glsl to use indexing.glslh with BufferMetadata/TextureMetadata UBOs, replacing legacy indexing_utils.h with its manual push-constant sizes, individual size/stride UBOs, and old indexing functions (nchwi_to_tidx, tidx_to_bufi, tidx_to_pos, tidx_to_posi) - Make concat_dim a specialization constant instead of a UBO parameter, allowing the compiler to eliminate dead branches and resolve indexing at pipeline creation time (follows the same pattern as gather's gather_dim) - Apply safe_idx helpers for all UBO ivec4/uvec4 access with spec-const-derived indices to avoid Adreno 740 SPIR-V compiler crash - Add uvec4 overload of safe_idx to indexing.glslh for BufferMetadata.sizes access - Update Concat.cpp to use graph.meta_ubo() instead of individual sizes/strides UBOs, add_storage_type_suffix() for kernel naming, and remove dead code in concat_pick_global_wg_size - Update update_concat_offset.glsl to use spec const concat_dim with safe_idx - Delete concat_utils.glslh (logic inlined into each shader) ghstack-source-id: 357266947 @exported-using-ghexport Differential Revision: [D97959283](https://our.internmc.facebook.com/intern/diff/D97959283/)
1 parent 1fcd773 commit 6f424e5

5 files changed

Lines changed: 112 additions & 197 deletions

File tree

‎backends/vulkan/runtime/graph/ops/glsl/concat_buffer.glsl‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,13 @@ ${define_required_extensions("buffer", DTYPE)}
1212

1313
#define PRECISION ${PRECISION}
1414

15-
#define VEC4_T ${texel_type(DTYPE)}
1615
#define T ${buffer_scalar_type(DTYPE)}
1716

1817
${define_active_storage_type("buffer")}
1918

2019
layout(std430) buffer;
2120

22-
#include "indexing_utils.h"
21+
#include "indexing.glslh"
2322

2423
${layout_declare_tensor(B, "rw", "t_out", DTYPE, "buffer")}
2524

@@ -28,70 +27,78 @@ $for i in range(NUM_INPUTS):
2827

2928
${layout_declare_tensor(B, "r", "t_concat_offset", "int", "buffer")}
3029

31-
${layout_declare_ubo(B, "int", "concat_dim")}
32-
33-
${layout_declare_ubo(B, "ivec4", "out_sizes")}
34-
${layout_declare_ubo(B, "ivec4", "out_strides")}
30+
${layout_declare_ubo(B, "BufferMetadata", "outp")}
3531

3632
$for i in range(NUM_INPUTS):
37-
${layout_declare_ubo(B, "ivec4", "inp" + str(i) + "_sizes")}
38-
${layout_declare_ubo(B, "ivec4", "inp" + str(i) + "_strides")}
39-
40-
${layout_declare_ubo(B, "int", "out_numel")}
33+
${layout_declare_ubo(B, "BufferMetadata", "inp" + str(i) + "p")}
4134

42-
${layout_declare_spec_const(C, "int", "out_layout", "DEFAULT_LAYOUT")}
43-
44-
const lowp ivec4 out_dim_order = unhash_dim_order(out_layout);
35+
${layout_declare_spec_const(C, "int", "concat_dim", "0")}
36+
${layout_declare_spec_const(C, "int", "out_layout", "CONTIG_LAYOUT_INT")}
4537

4638
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
4739

4840
#define NUM_INPUTS ${NUM_INPUTS}
4941

50-
#include "concat_utils.glslh"
51-
5242
/*
5343
* This shader template concatenates up to NUM_INPUT input tensors to the
5444
* output tensor along the concat_dim. Elements from the input tensor will
5545
* be inserted along the output's concat_dim starting at concat_offset.
5646
*/
5747
void main() {
58-
const int tid = ivec3(gl_GlobalInvocationID).x;
48+
const int tid = int(gl_GlobalInvocationID.x);
5949

6050
// The 1-3 input tensors are interpreted as one concatenated tensor ("volume")
6151
// along the concat_dim for the purposes of tensor indexing. Each thread is
6252
// responsible for reading one item from this volume and writing it to the
6353
// appropriate output location.
64-
ivec4 inp_volume_sizes = out_sizes;
65-
inp_volume_sizes[concat_dim] = total_concat_dim_numel();
54+
55+
// Compute inp_volume_sizes from output sizes, replacing concat_dim with the
56+
// sum of all input sizes along that dimension.
57+
ivec4 inp_volume_sizes = ivec4(outp.sizes[0]);
58+
int total_concat_dim = 0;
59+
$for i in range(NUM_INPUTS):
60+
total_concat_dim += int(safe_idx(inp${i}p.sizes[0], concat_dim));
61+
safe_set(inp_volume_sizes, concat_dim, total_concat_dim);
6662

6763
// Account for 0 size input tensors
6864
if (any(lessThanEqual(inp_volume_sizes, ivec4(0)))) {
6965
return;
7066
}
7167

72-
ivec4 inp_volume_tidx = nchwi_to_tidx(tid, inp_volume_sizes);
68+
// Decompose flat index into 4D tensor index (contiguous WHCN layout)
69+
TensorIndex4D inp_volume_tidx;
70+
{
71+
int remaining = tid;
72+
const int div_x = remaining / inp_volume_sizes.x;
73+
const int div_xy = div_x / inp_volume_sizes.y;
74+
inp_volume_tidx.data = ivec4(
75+
remaining % inp_volume_sizes.x,
76+
div_x % inp_volume_sizes.y,
77+
div_xy % inp_volume_sizes.z,
78+
div_xy / inp_volume_sizes.z);
79+
}
7380

7481
// bounds check
75-
if (any(greaterThanEqual(inp_volume_tidx, inp_volume_sizes))) {
82+
if (any(greaterThanEqual(inp_volume_tidx.data, inp_volume_sizes))) {
7683
return;
7784
}
7885

7986
int concat_offset = t_concat_offset[0];
8087

81-
ivec4 out_tidx = inp_volume_tidx;
82-
out_tidx[concat_dim] += concat_offset;
88+
TensorIndex4D out_tidx = inp_volume_tidx;
89+
out_tidx.data[concat_dim] += concat_offset;
8390

84-
const uint out_bufi = tidx_to_bufi(out_tidx, out_strides);
91+
const uint out_bufi = tensor4d_idx_to_linear_idx(outp, out_tidx);
8592

8693
// Go through the list of input tensors, and find which input this output
8794
// element should be read from.
8895
$for i in range(NUM_INPUTS):
89-
if (inp_volume_tidx[concat_dim] < inp${i}_sizes[concat_dim]) {
90-
int inp_bufi = tidx_to_bufi(inp_volume_tidx, inp${i}_strides);
96+
if (inp_volume_tidx.data[concat_dim] < int(safe_idx(inp${i}p.sizes[0], concat_dim))) {
97+
uint inp_bufi = tensor4d_idx_to_linear_idx(inp${i}p, inp_volume_tidx);
9198
t_out[out_bufi] = t_inp${i}[inp_bufi];
9299
return;
93100
}
94101
else {
95-
inp_volume_tidx[concat_dim] -= inp${i}_sizes[concat_dim];
102+
inp_volume_tidx.data[concat_dim] -= int(safe_idx(inp${i}p.sizes[0], concat_dim));
96103
}
97104
}

‎backends/vulkan/runtime/graph/ops/glsl/concat_texture.glsl‎

Lines changed: 63 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,18 @@
88

99
#version 450 core
1010

11+
${define_required_extensions("texture3d", DTYPE)}
12+
1113
#define PRECISION ${PRECISION}
1214

1315
#define VEC4_T ${texel_type(DTYPE)}
1416
#define T ${buffer_scalar_type(DTYPE)}
1517

16-
#define USING_TEXTURE3D
18+
${define_active_storage_type("texture3d")}
1719

1820
layout(std430) buffer;
1921

20-
#include "indexing_utils.h"
22+
#include "indexing.glslh"
2123

2224
${layout_declare_tensor(B, "rw", "t_out", DTYPE, "texture3d")}
2325

@@ -26,32 +28,23 @@ $for i in range(NUM_INPUTS):
2628

2729
${layout_declare_tensor(B, "r", "t_concat_offset", "int", "buffer")}
2830

29-
${layout_declare_ubo(B, "int", "concat_dim")}
31+
${layout_declare_ubo(B, "TextureMetadata", "outp")}
3032

31-
$in_metadata = ""
3233
$for i in range(NUM_INPUTS):
33-
$in_metadata += "ivec4 inp" + str(i) + "_sizes;\n"
34-
35-
layout(push_constant) uniform restrict Block {
36-
ivec4 out_sizes;
37-
${in_metadata}
38-
};
34+
${layout_declare_ubo(B, "TextureMetadata", "inp" + str(i) + "p")}
3935

40-
${layout_declare_spec_const(C, "int", "out_layout", "DEFAULT_LAYOUT")}
41-
const lowp ivec4 out_axis_map = unhash_axis_map(out_layout);
42-
const lowp int out_packed_dim = unhash_packed_dim(out_layout);
36+
${layout_declare_spec_const(C, "int", "concat_dim", "0")}
37+
${layout_declare_spec_const(C, "int", "out_layout", "CONTIG_LAYOUT_INT")}
4338

4439
$for i in range(NUM_INPUTS):
45-
${layout_declare_spec_const(C, "int", "inp" + str(i) + "_layout", "DEFAULT_LAYOUT")}
46-
const lowp ivec4 inp${i}_axis_map = unhash_axis_map(inp${i}_layout);
47-
const lowp int inp${i}_packed_dim = unhash_packed_dim(inp${i}_layout);
40+
${layout_declare_spec_const(C, "int", "inp" + str(i) + "_layout", "CONTIG_LAYOUT_INT")}
41+
42+
const int out_packed_dim = get_packed_dim(out_layout);
4843

4944
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
5045

5146
#define NUM_INPUTS ${NUM_INPUTS}
5247

53-
#include "concat_utils.glslh"
54-
5548
/*
5649
* This shader template concatenates up to NUM_INPUT input tensors to the
5750
* output tensor along the concat_dim. Elements from the input tensor will
@@ -62,109 +55,107 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
6255
* input tensor.
6356
*/
6457
void main() {
65-
const int tid = ivec3(gl_GlobalInvocationID).x;
58+
const int tid = int(gl_GlobalInvocationID.x);
6659

67-
// Sum of the sizes of all input tensors along the concat_dim
68-
const int concat_numel = total_concat_dim_numel();
69-
70-
// The 1-3 input tensors are interpreted as one concatenated tensor ("volume")
71-
// along the concat_dim for the purposes of tensor indexing. Each thread is
72-
// responsible for writing out 4 elements along the packed dim of the output
73-
// tensor by reading the source data from the input tensor(s).
74-
ivec4 inp_volume_sizes = out_sizes;
75-
inp_volume_sizes[concat_dim] = total_concat_dim_numel();
60+
// Compute inp_volume_sizes from output sizes, replacing concat_dim with the
61+
// sum of all input sizes along that dimension.
62+
ivec4 inp_volume_sizes = outp.sizes;
63+
int total_concat_dim = 0;
64+
$for i in range(NUM_INPUTS):
65+
total_concat_dim += safe_idx(inp${i}p.sizes, concat_dim);
66+
safe_set(inp_volume_sizes, concat_dim, total_concat_dim);
7667

7768
// Reconstruct inp_volume_texel_sizes from Concat.cpp
7869
ivec4 inp_volume_texel_sizes = inp_volume_sizes;
79-
inp_volume_texel_sizes[out_packed_dim] = DIV_UP_4(
80-
inp_volume_texel_sizes[out_packed_dim]
81-
) + 1;
70+
{
71+
int packed_size = inp_volume_texel_sizes[out_packed_dim];
72+
inp_volume_texel_sizes[out_packed_dim] = div_up_4(packed_size) + 1;
73+
}
8274

83-
// tensor index of the first element that will be read from the input volume
84-
ivec4 inp_volume_start_tidx = nchwi_to_tidx(tid, inp_volume_texel_sizes);
85-
inp_volume_start_tidx[out_packed_dim] = MUL_4(
86-
inp_volume_start_tidx[out_packed_dim]
87-
);
75+
// Decompose flat index into 4D tensor index (contiguous WHCN layout)
76+
TensorIndex4D inp_volume_start_tidx;
77+
{
78+
int remaining = tid;
79+
const int div_x = remaining / inp_volume_texel_sizes.x;
80+
const int div_xy = div_x / inp_volume_texel_sizes.y;
81+
inp_volume_start_tidx.data = ivec4(
82+
remaining % inp_volume_texel_sizes.x,
83+
div_x % inp_volume_texel_sizes.y,
84+
div_xy % inp_volume_texel_sizes.z,
85+
div_xy / inp_volume_texel_sizes.z);
86+
}
87+
inp_volume_start_tidx.data[out_packed_dim] =
88+
inp_volume_start_tidx.data[out_packed_dim] * 4;
8889

8990
int concat_offset = t_concat_offset[0];
9091

9192
// tensor index of the first element that will be written to the output tensor
92-
ivec4 out_write_start_tidx = inp_volume_start_tidx;
93-
out_write_start_tidx[concat_dim] += concat_offset;
93+
TensorIndex4D out_write_start_tidx = inp_volume_start_tidx;
94+
out_write_start_tidx.data[concat_dim] += concat_offset;
9495

95-
// To write to the the desired output element, we will need to load the texel
96+
// To write to the desired output element, we will need to load the texel
9697
// to which the element belongs. Calculate the tensor index of the first
9798
// element of that texel.
98-
ivec4 out_read_start_tidx = out_write_start_tidx;
99-
out_read_start_tidx[out_packed_dim] = ALIGN_DOWN_4(
100-
out_write_start_tidx[out_packed_dim]);
99+
TensorIndex4D out_read_start_tidx = out_write_start_tidx;
100+
out_read_start_tidx.data[out_packed_dim] =
101+
out_write_start_tidx.data[out_packed_dim] & ~3;
101102

102103
// bounds check
103-
if (any(greaterThanEqual(out_read_start_tidx, out_sizes))) {
104+
if (any(greaterThanEqual(out_read_start_tidx.data, outp.sizes))) {
104105
return;
105106
}
106107

107-
ivec3 out_pos = tidx_to_pos(
108-
out_read_start_tidx,
109-
out_sizes,
110-
out_axis_map,
111-
out_packed_dim
112-
);
108+
ivec3 out_pos = tensor4d_idx_to_texel_pos_simple(
109+
outp, out_read_start_tidx, out_layout);
113110

114111
VEC4_T out_texel = imageLoad(t_out, out_pos);
115112

116113
for (int comp = 0; comp < 4; ++comp) {
117-
ivec4 out_tidx = out_read_start_tidx;
118-
out_tidx[out_packed_dim] += comp;
119-
114+
TensorIndex4D out_tidx = out_read_start_tidx;
115+
out_tidx.data[out_packed_dim] += comp;
120116

121117
// It's possible that the current texel element has been written to as part
122118
// of the previous input batch; if so, then don't overwrite this texel
123119
// element
124-
if (out_tidx[concat_dim] < concat_offset) {
120+
if (out_tidx.data[concat_dim] < concat_offset) {
125121
continue;
126122
}
127123

128124
// Calculate the tidx of the input volume that corresponds to this output
129125
// element
130-
ivec4 inp_volume_tidx = out_tidx;
131-
inp_volume_tidx[concat_dim] -= concat_offset;
126+
TensorIndex4D inp_volume_tidx = out_tidx;
127+
inp_volume_tidx.data[concat_dim] -= concat_offset;
132128

133129
// go through the list of input tensors, and figure out which input this
134130
// output element should be read from.
135131
$for i in range(NUM_INPUTS):
136-
if (inp_volume_tidx[concat_dim] < inp${i}_sizes[concat_dim]) {
132+
if (inp_volume_tidx.data[concat_dim] < safe_idx(inp${i}p.sizes, concat_dim)) {
137133
// Special fast path case if, for the first output texel element, the
138134
// corresponding input element is at the start of the texel it belongs
139135
// to. In this case, the input texel can be written as-is to the output
140-
// texel. Also require that The entire input texel is valid and does not
136+
// texel. Also require that the entire input texel is valid and does not
141137
// contain any padding elements.
138+
const int inp${i}_packed_dim = get_packed_dim(inp${i}_layout);
142139
if (comp == 0 &&
143-
out_tidx[out_packed_dim] % 4 == 0 &&
144-
inp_volume_tidx[inp${i}_packed_dim] % 4 == 0 &&
145-
inp_volume_tidx[inp${i}_packed_dim] + 3 < inp${i}_sizes[inp${i}_packed_dim]) {
146-
const ivec3 in_pos = tidx_to_pos(
147-
inp_volume_tidx,
148-
inp${i}_sizes,
149-
inp${i}_axis_map,
150-
inp${i}_packed_dim);
140+
out_tidx.data[out_packed_dim] % 4 == 0 &&
141+
inp_volume_tidx.data[inp${i}_packed_dim] % 4 == 0 &&
142+
inp_volume_tidx.data[inp${i}_packed_dim] + 3 < safe_idx(inp${i}p.sizes, inp${i}_packed_dim)) {
143+
const ivec3 in_pos = tensor4d_idx_to_texel_pos_simple(
144+
inp${i}p, inp_volume_tidx, inp${i}_layout);
151145

152146
out_texel = texelFetch(t_inp${i}, in_pos, 0);
153147
break;
154148
}
155149

156150
// Otherwise, locate the specific input element required
157-
const ivec4 in_posi = tidx_to_posi(
158-
inp_volume_tidx,
159-
inp${i}_sizes,
160-
inp${i}_axis_map,
161-
inp${i}_packed_dim);
151+
const TextureElementIndex in_elem = tensor4d_idx_to_texture_element_idx_simple(
152+
inp${i}p, inp_volume_tidx, inp${i}_layout);
162153

163-
out_texel[comp] = texelFetch(t_inp${i}, in_posi.xyz, 0)[in_posi.w];
154+
out_texel[comp] = texelFetch(t_inp${i}, in_elem.pos, 0)[in_elem.comp];
164155
continue;
165156
}
166157
else {
167-
inp_volume_tidx[concat_dim] -= inp${i}_sizes[concat_dim];
158+
inp_volume_tidx.data[concat_dim] -= safe_idx(inp${i}p.sizes, concat_dim);
168159
}
169160
}
170161

‎backends/vulkan/runtime/graph/ops/glsl/concat_utils.glslh‎

Lines changed: 0 additions & 33 deletions
This file was deleted.

0 commit comments

Comments
 (0)