88
99#version 450 core
1010
11+ ${define_required_extensions("texture3d", DTYPE)}
12+
1113#define PRECISION ${PRECISION}
1214
1315#define VEC4_T ${texel_type(DTYPE)}
1416#define T ${buffer_scalar_type(DTYPE)}
1517
16- #define USING_TEXTURE3D
18+ ${define_active_storage_type("texture3d")}
1719
1820layout (std430) buffer ;
1921
20- #include "indexing_utils.h "
22+ #include "indexing.glslh "
2123
2224${layout_declare_tensor(B, "rw", "t_out", DTYPE, "texture3d")}
2325
@@ -26,32 +28,23 @@ $for i in range(NUM_INPUTS):
2628
2729${layout_declare_tensor(B, "r", "t_concat_offset", "int ", "buffer ")}
2830
29- ${layout_declare_ubo(B, "int ", "concat_dim ")}
31+ ${layout_declare_ubo(B, "TextureMetadata ", "outp ")}
3032
31- $in_metadata = ""
3233$for i in range(NUM_INPUTS):
33- $in_metadata += "ivec4 inp" + str(i) + "_sizes;\n"
34-
35- layout (push_constant) uniform restrict Block {
36- ivec4 out_sizes;
37- ${in_metadata}
38- };
34+ ${layout_declare_ubo(B, "TextureMetadata", "inp" + str(i) + "p")}
3935
40- ${layout_declare_spec_const(C, "int ", "out_layout", "DEFAULT_LAYOUT")}
41- const lowp ivec4 out_axis_map = unhash_axis_map(out_layout);
42- const lowp int out_packed_dim = unhash_packed_dim(out_layout);
36+ ${layout_declare_spec_const(C, "int ", "concat_dim", "0 ")}
37+ ${layout_declare_spec_const(C, "int ", "out_layout", "CONTIG_LAYOUT_INT")}
4338
4439$for i in range(NUM_INPUTS):
45- ${layout_declare_spec_const(C, "int ", "inp" + str(i) + "_layout", "DEFAULT_LAYOUT ")}
46- const lowp ivec4 inp${i}_axis_map = unhash_axis_map(inp${i}_layout);
47- const lowp int inp${i}_packed_dim = unhash_packed_dim(inp${i}_layout );
40+ ${layout_declare_spec_const(C, "int ", "inp" + str(i) + "_layout", "CONTIG_LAYOUT_INT ")}
41+
42+ const int out_packed_dim = get_packed_dim(out_layout );
4843
4944layout (local_size_x_id = 0 , local_size_y_id = 1 , local_size_z_id = 2 ) in ;
5045
5146#define NUM_INPUTS ${NUM_INPUTS}
5247
53- #include "concat_utils.glslh"
54-
5548/*
5649 * This shader template concatenates up to NUM_INPUT input tensors to the
5750 * output tensor along the concat_dim. Elements from the input tensor will
@@ -62,109 +55,107 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
6255 * input tensor.
6356 */
6457void main() {
65- const int tid = ivec3 (gl_GlobalInvocationID).x ;
58+ const int tid = int (gl_GlobalInvocationID.x) ;
6659
67- // Sum of the sizes of all input tensors along the concat_dim
68- const int concat_numel = total_concat_dim_numel();
69-
70- // The 1-3 input tensors are interpreted as one concatenated tensor ("volume")
71- // along the concat_dim for the purposes of tensor indexing. Each thread is
72- // responsible for writing out 4 elements along the packed dim of the output
73- // tensor by reading the source data from the input tensor(s).
74- ivec4 inp_volume_sizes = out_sizes;
75- inp_volume_sizes[concat_dim] = total_concat_dim_numel();
60+ // Compute inp_volume_sizes from output sizes, replacing concat_dim with the
61+ // sum of all input sizes along that dimension.
62+ ivec4 inp_volume_sizes = outp.sizes;
63+ int total_concat_dim = 0 ;
64+ $for i in range(NUM_INPUTS):
65+ total_concat_dim += safe_idx(inp${i}p.sizes, concat_dim);
66+ safe_set(inp_volume_sizes, concat_dim, total_concat_dim);
7667
7768 // Reconstruct inp_volume_texel_sizes from Concat.cpp
7869 ivec4 inp_volume_texel_sizes = inp_volume_sizes;
79- inp_volume_texel_sizes[out_packed_dim] = DIV_UP_4(
80- inp_volume_texel_sizes[out_packed_dim]
81- ) + 1 ;
70+ {
71+ int packed_size = inp_volume_texel_sizes[out_packed_dim];
72+ inp_volume_texel_sizes[out_packed_dim] = div_up_4(packed_size) + 1 ;
73+ }
8274
83- // tensor index of the first element that will be read from the input volume
84- ivec4 inp_volume_start_tidx = nchwi_to_tidx(tid, inp_volume_texel_sizes);
85- inp_volume_start_tidx[out_packed_dim] = MUL_4(
86- inp_volume_start_tidx[out_packed_dim]
87- );
75+ // Decompose flat index into 4D tensor index (contiguous WHCN layout)
76+ TensorIndex4D inp_volume_start_tidx;
77+ {
78+ int remaining = tid;
79+ const int div_x = remaining / inp_volume_texel_sizes.x;
80+ const int div_xy = div_x / inp_volume_texel_sizes.y;
81+ inp_volume_start_tidx.data = ivec4 (
82+ remaining % inp_volume_texel_sizes.x,
83+ div_x % inp_volume_texel_sizes.y,
84+ div_xy % inp_volume_texel_sizes.z,
85+ div_xy / inp_volume_texel_sizes.z);
86+ }
87+ inp_volume_start_tidx.data[out_packed_dim] =
88+ inp_volume_start_tidx.data[out_packed_dim] * 4 ;
8889
8990 int concat_offset = t_concat_offset[0 ];
9091
9192 // tensor index of the first element that will be written to the output tensor
92- ivec4 out_write_start_tidx = inp_volume_start_tidx;
93- out_write_start_tidx[concat_dim] += concat_offset;
93+ TensorIndex4D out_write_start_tidx = inp_volume_start_tidx;
94+ out_write_start_tidx.data [concat_dim] += concat_offset;
9495
95- // To write to the the desired output element, we will need to load the texel
96+ // To write to the desired output element, we will need to load the texel
9697 // to which the element belongs. Calculate the tensor index of the first
9798 // element of that texel.
98- ivec4 out_read_start_tidx = out_write_start_tidx;
99- out_read_start_tidx[out_packed_dim] = ALIGN_DOWN_4(
100- out_write_start_tidx[out_packed_dim]) ;
99+ TensorIndex4D out_read_start_tidx = out_write_start_tidx;
100+ out_read_start_tidx.data [out_packed_dim] =
101+ out_write_start_tidx.data [out_packed_dim] & ~ 3 ;
101102
102103 // bounds check
103- if (any (greaterThanEqual (out_read_start_tidx, out_sizes ))) {
104+ if (any (greaterThanEqual (out_read_start_tidx.data, outp.sizes ))) {
104105 return ;
105106 }
106107
107- ivec3 out_pos = tidx_to_pos(
108- out_read_start_tidx,
109- out_sizes,
110- out_axis_map,
111- out_packed_dim
112- );
108+ ivec3 out_pos = tensor4d_idx_to_texel_pos_simple(
109+ outp, out_read_start_tidx, out_layout);
113110
114111 VEC4_T out_texel = imageLoad(t_out, out_pos);
115112
116113 for (int comp = 0 ; comp < 4 ; ++ comp) {
117- ivec4 out_tidx = out_read_start_tidx;
118- out_tidx[out_packed_dim] += comp;
119-
114+ TensorIndex4D out_tidx = out_read_start_tidx;
115+ out_tidx.data[out_packed_dim] += comp;
120116
121117 // It's possible that the current texel element has been written to as part
122118 // of the previous input batch; if so, then don't overwrite this texel
123119 // element
124- if (out_tidx[concat_dim] < concat_offset) {
120+ if (out_tidx.data [concat_dim] < concat_offset) {
125121 continue ;
126122 }
127123
128124 // Calculate the tidx of the input volume that corresponds to this output
129125 // element
130- ivec4 inp_volume_tidx = out_tidx;
131- inp_volume_tidx[concat_dim] -= concat_offset;
126+ TensorIndex4D inp_volume_tidx = out_tidx;
127+ inp_volume_tidx.data [concat_dim] -= concat_offset;
132128
133129 // go through the list of input tensors, and figure out which input this
134130 // output element should be read from.
135131 $for i in range(NUM_INPUTS):
136- if (inp_volume_tidx[concat_dim] < inp${i}_sizes[ concat_dim] ) {
132+ if (inp_volume_tidx.data [concat_dim] < safe_idx( inp${i}p.sizes, concat_dim) ) {
137133 // Special fast path case if, for the first output texel element, the
138134 // corresponding input element is at the start of the texel it belongs
139135 // to. In this case, the input texel can be written as-is to the output
140- // texel. Also require that The entire input texel is valid and does not
136+ // texel. Also require that the entire input texel is valid and does not
141137 // contain any padding elements.
138+ const int inp${i}_packed_dim = get_packed_dim(inp${i}_layout);
142139 if (comp == 0 &&
143- out_tidx[out_packed_dim] % 4 == 0 &&
144- inp_volume_tidx[inp${i}_packed_dim] % 4 == 0 &&
145- inp_volume_tidx[inp${i}_packed_dim] + 3 < inp${i}_sizes[inp${i}_packed_dim]) {
146- const ivec3 in_pos = tidx_to_pos(
147- inp_volume_tidx,
148- inp${i}_sizes,
149- inp${i}_axis_map,
150- inp${i}_packed_dim);
140+ out_tidx.data[out_packed_dim] % 4 == 0 &&
141+ inp_volume_tidx.data[inp${i}_packed_dim] % 4 == 0 &&
142+ inp_volume_tidx.data[inp${i}_packed_dim] + 3 < safe_idx(inp${i}p.sizes, inp${i}_packed_dim)) {
143+ const ivec3 in_pos = tensor4d_idx_to_texel_pos_simple(
144+ inp${i}p, inp_volume_tidx, inp${i}_layout);
151145
152146 out_texel = texelFetch(t_inp${i}, in_pos, 0 );
153147 break ;
154148 }
155149
156150 // Otherwise, locate the specific input element required
157- const ivec4 in_posi = tidx_to_posi(
158- inp_volume_tidx,
159- inp${i}_sizes,
160- inp${i}_axis_map,
161- inp${i}_packed_dim);
151+ const TextureElementIndex in_elem = tensor4d_idx_to_texture_element_idx_simple(
152+ inp${i}p, inp_volume_tidx, inp${i}_layout);
162153
163- out_texel[comp] = texelFetch(t_inp${i}, in_posi.xyz , 0 )[in_posi.w ];
154+ out_texel[comp] = texelFetch(t_inp${i}, in_elem.pos , 0 )[in_elem.comp ];
164155 continue ;
165156 }
166157 else {
167- inp_volume_tidx[concat_dim] -= inp${i}_sizes[ concat_dim] ;
158+ inp_volume_tidx.data [concat_dim] -= safe_idx( inp${i}p.sizes, concat_dim) ;
168159 }
169160 }
170161
0 commit comments