|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +#version 450 core |
| 10 | + |
| 11 | +${define_required_extensions("buffer", DTYPE)} |
| 12 | + |
| 13 | +#extension GL_EXT_control_flow_attributes : require |
| 14 | +#extension GL_EXT_integer_dot_product : require |
| 15 | + |
| 16 | +#define PRECISION ${PRECISION} |
| 17 | +#define VEC4_T ${texel_load_type(DTYPE, "buffer")} |
| 18 | +#define T int |
| 19 | + |
| 20 | +$if WEIGHT_STORAGE == "buffer": |
| 21 | + #define WEIGHT_BUFFER |
| 22 | + |
| 23 | +#define TILE_K4 ${TILE_K4} |
| 24 | +#define TILE_N4 ${TILE_N4} |
| 25 | + |
| 26 | +#define TILE_M4 1 |
| 27 | +#define TILE_M 1 |
| 28 | +#define TILE_K ${TILE_K4 * 4} |
| 29 | +#define TILE_N ${TILE_N4 * 4} |
| 30 | + |
| 31 | +#define WGS ${WGS} |
| 32 | + |
| 33 | +layout(std430) buffer; |
| 34 | + |
| 35 | +// Scalar int arrays for 4W packed int8 input/output |
| 36 | +${layout_declare_tensor(B, "w", "t_packed_int8_output", "int", "buffer")} |
| 37 | +${layout_declare_tensor(B, "r", "t_packed_int8_input", "int", "buffer")} |
| 38 | +// Weight uses ivec4 (same format as q8ta_linear) |
| 39 | +${layout_declare_tensor(B, "r", "t_packed_int8_weight", "int", WEIGHT_STORAGE, is_scalar_array=False)} |
| 40 | +${layout_declare_tensor(B, "r", "t_weight_sums", "int", "buffer", is_scalar_array=False)} |
| 41 | +${layout_declare_tensor(B, "r", "t_weight_scales", DTYPE, "buffer", is_scalar_array=False)} |
| 42 | +${layout_declare_tensor(B, "r", "t_bias", DTYPE, "buffer", is_scalar_array=False)} |
| 43 | + |
| 44 | +${layout_declare_spec_const(C, "int", "apply_bias", "0")} |
| 45 | +${layout_declare_spec_const(C, "int", "activation_type", "0")} |
| 46 | + |
| 47 | +${layout_declare_ubo(B, "ivec4", "output_sizes")} |
| 48 | +${layout_declare_ubo(B, "ivec4", "input_sizes")} |
| 49 | + |
| 50 | +layout(push_constant) uniform restrict Block { |
| 51 | + float input_scale; |
| 52 | + int input_zp; |
| 53 | + float output_inv_scale; |
| 54 | + int output_zp; |
| 55 | +}; |
| 56 | + |
| 57 | +layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; |
| 58 | + |
| 59 | +#include "common.glslh" |
| 60 | +#include "linear_int8_weight_tile_load.glslh" |
| 61 | +#include "linear_fp_output_tile_int8_int8_compute.glslh" |
| 62 | +#include "linear_fp_weight_scales_load.glslh" |
| 63 | +#include "linear_int_weight_sums_load.glslh" |
| 64 | +#include "linear_fp_bias_load.glslh" |
| 65 | + |
| 66 | +shared Int32Accum partial_accums[WGS]; |
| 67 | + |
| 68 | +void main() { |
| 69 | + const int lid = int(gl_LocalInvocationID.z); |
| 70 | + const int n4 = int(gl_GlobalInvocationID.x) * TILE_N4; |
| 71 | + |
| 72 | + const int n = mul_4(n4); |
| 73 | + |
| 74 | + const int K4 = div_up_4(input_sizes.x); |
| 75 | + const int N4 = div_up_4(output_sizes.x); |
| 76 | + |
| 77 | + if (n >= output_sizes.x) { |
| 78 | + return; |
| 79 | + } |
| 80 | + |
| 81 | + Int32Accum out_accum; |
| 82 | + initialize(out_accum); |
| 83 | + |
| 84 | + Int8WeightTile int8_weight_tile; |
| 85 | + |
| 86 | + for (int k4 = lid; k4 < K4; k4 += WGS) { |
| 87 | + // Load one packed int32 from the 4W input buffer. Each int32 contains |
| 88 | + // 4 int8 values at k=k4*4..k4*4+3. |
| 89 | + const int packed_input = t_packed_int8_input[k4]; |
| 90 | + |
| 91 | + load_int8_weight_tile(int8_weight_tile, n4, k4, N4); |
| 92 | + |
| 93 | + // Accumulate dot products of the input int8x4 with each weight int8x4 |
| 94 | + [[unroll]] for (int n = 0; n < TILE_N; ++n) { |
| 95 | + const int tile_n4 = div_4(n); |
| 96 | + const int n4i = mod_4(n); |
| 97 | + out_accum.data[0][tile_n4][n4i] = dotPacked4x8AccSatEXT( |
| 98 | + packed_input, |
| 99 | + int8_weight_tile.data[0][tile_n4][n4i], |
| 100 | + out_accum.data[0][tile_n4][n4i]); |
| 101 | + } |
| 102 | + } |
| 103 | + |
| 104 | + partial_accums[lid] = out_accum; |
| 105 | + |
| 106 | + memoryBarrierShared(); |
| 107 | + barrier(); |
| 108 | + |
| 109 | + // Only the first thread writes the result |
| 110 | + if (lid == 0) { |
| 111 | + for (int i = 1; i < WGS; ++i) { |
| 112 | + [[unroll]] for (int tile_n4 = 0; tile_n4 < TILE_N4; ++tile_n4) { |
| 113 | + out_accum.data[0][tile_n4] += |
| 114 | + partial_accums[i].data[0][tile_n4]; |
| 115 | + } |
| 116 | + } |
| 117 | + |
| 118 | + FPPerOutChannelParams weight_scales_tile; |
| 119 | + load_weight_scales_tile(weight_scales_tile, n4); |
| 120 | + |
| 121 | + IntPerOutChannelParams weight_sums_tile; |
| 122 | + load_weight_sums_tile(weight_sums_tile, n4); |
| 123 | + |
| 124 | + FPOutTile out_tile; |
| 125 | + initialize(out_tile); |
| 126 | + |
| 127 | + if (apply_bias > 0) { |
| 128 | + FPPerOutChannelParams bias_tile; |
| 129 | + load_bias_tile(bias_tile, n4); |
| 130 | + |
| 131 | + accumulate_out_tile_with_int_accum( |
| 132 | + out_tile, |
| 133 | + out_accum, |
| 134 | + input_scale, |
| 135 | + input_zp, |
| 136 | + weight_sums_tile, |
| 137 | + weight_scales_tile, |
| 138 | + bias_tile); |
| 139 | + } else { |
| 140 | + accumulate_out_tile_with_int_accum( |
| 141 | + out_tile, |
| 142 | + out_accum, |
| 143 | + input_scale, |
| 144 | + input_zp, |
| 145 | + weight_sums_tile, |
| 146 | + weight_scales_tile); |
| 147 | + } |
| 148 | + |
| 149 | + // Apply ReLU if enabled |
| 150 | + if (activation_type > 0) { |
| 151 | + [[unroll]] for (int tile_n4 = 0; tile_n4 < TILE_N4; ++tile_n4) { |
| 152 | + out_tile.data[0][tile_n4] = max(out_tile.data[0][tile_n4], vec4(0.0)); |
| 153 | + } |
| 154 | + } |
| 155 | + |
| 156 | + // Quantize and write to scalar int[] buffer. Each int32 at position n4 |
| 157 | + // contains 4 packed int8 output values for channels n4*4..n4*4+3. |
| 158 | + [[unroll]] for (int tile_n4 = 0; tile_n4 < TILE_N4; ++tile_n4) { |
| 159 | + if (n4 + tile_n4 < N4) { |
| 160 | + t_packed_int8_output[n4 + tile_n4] = quantize_and_pack( |
| 161 | + out_tile.data[0][tile_n4], output_inv_scale, output_zp); |
| 162 | + } |
| 163 | + } |
| 164 | + } |
| 165 | +} |
0 commit comments