Skip to content

Commit 6d0b933

Browse files
authored
embedding_q4gsw: support is_linear_weight packing (pytorch#20639)
Summary: The WebGPU `embedding_q4gsw` op failed to load (`Error::DelegateInvalidCompatibility`, error 48): `WebGPUGraph::build()` threw `is_linear_weight=true is unsupported` and `WebGPUBackend.cpp` returned error 48 at `load_forward`. The `quantized_decomposed.embedding_4bit` and torchao fusions in `backends/vulkan/patterns/quantized_embedding.py` repack the embedding weight into the 4-bit linear-layer nibble convention (low nibble = even dim, high nibble = odd dim) and emit `et_vk.embedding_q4gsw.default(..., is_linear_weight=true)` — always on OSS main, and for tied (embedding/LM-head-shared) weights via `_detect_tied_linear_weight`. The Vulkan runtime supports both packings via its compile-time `_linear_weight` shader variant; the WebGPU runtime rejected `true` outright, so any model whose embedding took that path failed to delegate. This teaches the WebGPU runtime both packings. The handler now forwards `is_linear_weight` to the shader through the uniform (the spare `_pad` field is repurposed; struct size unchanged), and `embedding_q4gsw.wgsl` selects the nibble with `use_high = is_even != is_linear_weight`. The `is_linear_weight=false` path is byte-identical to before. Differential Revision: D110211746
1 parent e095c57 commit 6d0b933

3 files changed

Lines changed: 17 additions & 15 deletions

File tree

‎backends/webgpu/runtime/ops/embedding_q4gsw/EmbeddingQ4gsw.cpp‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ struct EmbeddingParams {
3030
uint32_t groups_per_row;
3131
uint32_t bytes_per_row;
3232
uint32_t total_blocks;
33-
uint32_t _pad;
33+
uint32_t is_linear_weight;
3434
};
3535
static_assert(
3636
sizeof(EmbeddingParams) == 32,
@@ -60,7 +60,8 @@ void embedding_q4gsw_impl(WebGPUGraph& graph, const std::vector<int>& args) {
6060
const auto& indices = graph.get_tensor(indices_id);
6161
const auto& out = graph.get_tensor(out_id);
6262

63-
// Only the flat weight path is supported (linear-block unsupported).
63+
// is_linear_weight selects the nibble packing (false: even dim = high nibble;
64+
// true: even dim = low nibble). The shader handles both via a uniform.
6465
bool is_linear = false;
6566
if (graph.get_value_type(is_linear_weight_id) ==
6667
WebGPUGraph::ValueType::Bool) {
@@ -73,10 +74,6 @@ void embedding_q4gsw_impl(WebGPUGraph& graph, const std::vector<int>& args) {
7374
throw std::runtime_error(
7475
"WebGPU embedding_q4gsw: is_linear_weight must be Bool or Int");
7576
}
76-
if (is_linear) {
77-
throw std::runtime_error(
78-
"WebGPU embedding_q4gsw: is_linear_weight=true is unsupported");
79-
}
8077

8178
if (weight.dims.size() < 2 || scales.dims.size() < 2 || out.dims.empty() ||
8279
indices.dims.empty()) {
@@ -150,6 +147,7 @@ void embedding_q4gsw_impl(WebGPUGraph& graph, const std::vector<int>& args) {
150147
params.groups_per_row = groups_per_row;
151148
params.bytes_per_row = bytes_per_row;
152149
params.total_blocks = static_cast<uint32_t>(total_blocks);
150+
params.is_linear_weight = is_linear ? 1u : 0u;
153151

154152
WGPUBufferDescriptor uniform_desc = {};
155153
uniform_desc.size = sizeof(EmbeddingParams);

‎backends/webgpu/runtime/ops/embedding_q4gsw/embedding_q4gsw.wgsl‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ struct Params {
1111
groups_per_row: u32,
1212
bytes_per_row: u32,
1313
total_blocks: u32,
14-
_pad: u32,
14+
is_linear_weight: u32,
1515
}
1616
@group(0) @binding(4) var<uniform> params: Params;
1717

@@ -37,11 +37,13 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
3737
let byte_idx = row_byte_base + (dim >> 1u);
3838
let word = t_weight[byte_idx >> 2u];
3939
let b = (word >> ((byte_idx & 3u) * 8u)) & 0xFFu;
40+
// Nibble packing depends on is_linear_weight: non-linear maps even dim ->
41+
// high nibble / odd -> low; linear maps even -> low / odd -> high.
4042
var nib: u32;
41-
if ((dim & 1u) == 0u) {
42-
nib = (b >> 4u) & 0x0Fu; // even dim -> high nibble
43+
if (((dim & 1u) == 0u) != (params.is_linear_weight != 0u)) {
44+
nib = (b >> 4u) & 0x0Fu; // high nibble
4345
} else {
44-
nib = b & 0x0Fu; // odd dim -> low nibble
46+
nib = b & 0x0Fu; // low nibble
4547
}
4648
let q = f32(i32(nib) - 8); // +8-shifted on pack; recover signed [-8,7]
4749
let scale = t_scales[token * params.groups_per_row + dim / params.group_size];

‎backends/webgpu/runtime/ops/embedding_q4gsw/embedding_q4gsw_wgsl.h‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
namespace executorch::backends::webgpu {
1414

1515
// @generated from embedding_q4gsw.wgsl - DO NOT EDIT.
16-
// wgsl-sha256: 1fec9ed315696a88bb7db6c16454fc80e08ff73b0e39720b54515fda4ee1ef7c
16+
// wgsl-sha256: 94da1061b49b62556a79020182a4989439a7c51f919e83d577536c5b6d25f487
1717
inline constexpr const char* kEmbeddingQ4gswWGSL = R"(
1818
@group(0) @binding(0) var<storage, read_write> t_out: array<f32>;
1919
@group(0) @binding(1) var<storage, read> t_indices: array<i32>;
@@ -28,7 +28,7 @@ struct Params {
2828
groups_per_row: u32,
2929
bytes_per_row: u32,
3030
total_blocks: u32,
31-
_pad: u32,
31+
is_linear_weight: u32,
3232
}
3333
@group(0) @binding(4) var<uniform> params: Params;
3434
@@ -54,11 +54,13 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
5454
let byte_idx = row_byte_base + (dim >> 1u);
5555
let word = t_weight[byte_idx >> 2u];
5656
let b = (word >> ((byte_idx & 3u) * 8u)) & 0xFFu;
57+
// Nibble packing depends on is_linear_weight: non-linear maps even dim ->
58+
// high nibble / odd -> low; linear maps even -> low / odd -> high.
5759
var nib: u32;
58-
if ((dim & 1u) == 0u) {
59-
nib = (b >> 4u) & 0x0Fu; // even dim -> high nibble
60+
if (((dim & 1u) == 0u) != (params.is_linear_weight != 0u)) {
61+
nib = (b >> 4u) & 0x0Fu; // high nibble
6062
} else {
61-
nib = b & 0x0Fu; // odd dim -> low nibble
63+
nib = b & 0x0Fu; // low nibble
6264
}
6365
let q = f32(i32(nib) - 8); // +8-shifted on pack; recover signed [-8,7]
6466
let scale = t_scales[token * params.groups_per_row + dim / params.group_size];

0 commit comments

Comments
 (0)