|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +#pragma once |
| 10 | + |
| 11 | +#include <cstdint> |
| 12 | + |
| 13 | +namespace executorch::backends::webgpu { |
| 14 | + |
| 15 | +// @generated from q4gsw_linear_coop4_bicol.wgsl - DO NOT EDIT. |
| 16 | +// wgsl-sha256: 44c53ca7879f7a3382a45703a67cbc6fc221ecbcada58f8c413df50abbc6e544 |
| 17 | +inline constexpr const char* kQ4gswLinearCoop4BicolWGSL = R"( |
| 18 | +@group(0) @binding(0) var<storage, read_write> t_out: array<f32>; |
| 19 | +@group(0) @binding(1) var<storage, read> t_input: array<f32>; |
| 20 | +@group(0) @binding(2) var<storage, read> t_weight: array<u32>; |
| 21 | +@group(0) @binding(3) var<storage, read> t_scales: array<f32>; |
| 22 | +@group(0) @binding(4) var<storage, read> t_bias: array<f32>; |
| 23 | +
|
| 24 | +struct Params { |
| 25 | + M: u32, |
| 26 | + N: u32, |
| 27 | + K: u32, |
| 28 | + K_packed: u32, |
| 29 | + group_size: u32, |
| 30 | + padded_N: u32, |
| 31 | + has_bias: u32, |
| 32 | + _pad: u32, |
| 33 | +} |
| 34 | +@group(0) @binding(5) var<uniform> params: Params; |
| 35 | +
|
| 36 | +// Cooperative-over-K GEMV, 2 output columns/workgroup; input read once, reused. |
| 37 | +const WG: u32 = 64u; |
| 38 | +var<workgroup> partial: array<vec2<f32>, WG>; |
| 39 | +
|
| 40 | +@compute @workgroup_size(WG, 1, 1) |
| 41 | +fn main( |
| 42 | + @builtin(workgroup_id) wid: vec3<u32>, |
| 43 | + @builtin(num_workgroups) ngrp: vec3<u32>, |
| 44 | + @builtin(local_invocation_id) lid: vec3<u32>) { |
| 45 | + let num_pairs = (params.N + 1u) >> 1u; // ceil(N/2) pairs (M == 1 decode) |
| 46 | + let stride = ngrp.x; |
| 47 | + let num_words = params.K >> 3u; // K / 8 words per row |
| 48 | + let row_words = params.K_packed >> 2u; // u32s per weight row (= K/8) |
| 49 | + var p = wid.x; |
| 50 | + loop { |
| 51 | + if (p >= num_pairs) { |
| 52 | + break; |
| 53 | + } |
| 54 | + let col0 = p << 1u; |
| 55 | + let col1 = col0 + 1u; |
| 56 | + let has1 = col1 < params.N; |
| 57 | + let wbase0 = col0 * row_words; |
| 58 | + let wbase1 = col1 * row_words; // load guarded by has1 |
| 59 | +
|
| 60 | + var acc0: f32 = 0.0; |
| 61 | + var acc1: f32 = 0.0; |
| 62 | + var w: u32 = lid.x; |
| 63 | + loop { |
| 64 | + if (w >= num_words) { |
| 65 | + break; |
| 66 | + } |
| 67 | + let k0 = w << 3u; |
| 68 | + let ib = k0; // in_base = 0 (m == 0 for M == 1 decode) |
| 69 | + let sidx = (k0 / params.group_size) * params.padded_N; |
| 70 | + let word0 = t_weight[wbase0 + w]; |
| 71 | + let scale0 = t_scales[sidx + col0]; |
| 72 | + var word1: u32 = 0u; |
| 73 | + var scale1: f32 = 0.0; |
| 74 | + if (has1) { |
| 75 | + word1 = t_weight[wbase1 + w]; |
| 76 | + scale1 = t_scales[sidx + col1]; |
| 77 | + } |
| 78 | + for (var bi: u32 = 0u; bi < 4u; bi = bi + 1u) { |
| 79 | + let kk = bi << 1u; |
| 80 | + let in0 = t_input[ib + kk]; // shared across both columns |
| 81 | + let in1 = t_input[ib + kk + 1u]; |
| 82 | + let b0 = (word0 >> (bi * 8u)) & 0xFFu; |
| 83 | + acc0 = acc0 + in0 * f32(i32(b0 & 0x0Fu) - 8) * scale0; |
| 84 | + acc0 = acc0 + in1 * f32(i32((b0 >> 4u) & 0x0Fu) - 8) * scale0; |
| 85 | + let b1 = (word1 >> (bi * 8u)) & 0xFFu; |
| 86 | + acc1 = acc1 + in0 * f32(i32(b1 & 0x0Fu) - 8) * scale1; |
| 87 | + acc1 = acc1 + in1 * f32(i32((b1 >> 4u) & 0x0Fu) - 8) * scale1; |
| 88 | + } |
| 89 | + w = w + WG; |
| 90 | + } |
| 91 | +
|
| 92 | + partial[lid.x] = vec2<f32>(acc0, acc1); |
| 93 | + workgroupBarrier(); |
| 94 | + var s: u32 = WG >> 1u; |
| 95 | + loop { |
| 96 | + if (s == 0u) { |
| 97 | + break; |
| 98 | + } |
| 99 | + if (lid.x < s) { |
| 100 | + partial[lid.x] = partial[lid.x] + partial[lid.x + s]; |
| 101 | + } |
| 102 | + workgroupBarrier(); |
| 103 | + s = s >> 1u; |
| 104 | + } |
| 105 | + if (lid.x == 0u) { |
| 106 | + var o0 = partial[0].x; |
| 107 | + var o1 = partial[0].y; |
| 108 | + if (params.has_bias != 0u) { |
| 109 | + o0 = o0 + t_bias[col0]; |
| 110 | + if (has1) { |
| 111 | + o1 = o1 + t_bias[col1]; |
| 112 | + } |
| 113 | + } |
| 114 | + t_out[col0] = o0; |
| 115 | + if (has1) { |
| 116 | + t_out[col1] = o1; |
| 117 | + } |
| 118 | + } |
| 119 | + workgroupBarrier(); |
| 120 | + p = p + stride; |
| 121 | + } |
| 122 | +} |
| 123 | +)"; |
| 124 | + |
| 125 | +inline constexpr uint32_t kQ4gswLinearCoop4BicolWorkgroupSizeX = 64; |
| 126 | +inline constexpr uint32_t kQ4gswLinearCoop4BicolWorkgroupSizeY = 1; |
| 127 | +inline constexpr uint32_t kQ4gswLinearCoop4BicolWorkgroupSizeZ = 1; |
| 128 | + |
| 129 | +} // namespace executorch::backends::webgpu |
0 commit comments