Skip to content

Commit fd82979

Browse files
[executorch][native-vk] Lower q4 embeddings at load time (pytorch#23474)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23441 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/134/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/134/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/133/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/134/orig Differential Revision: [D121864328](https://our.internmc.facebook.com/intern/diff/D121864328/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent ab7a616 commit fd82979

6 files changed

Lines changed: 501 additions & 1 deletion

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include <executorch/backends/native/runtime/graph/Value.h>
3838
#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h>
3939
#include <executorch/backends/native/runtime/vulkan/VulkanConstantMaterializationTracker.h>
40+
#include <executorch/backends/native/runtime/vulkan/passes/FuseQuantizedEmbedding.h>
4041
#include <executorch/backends/native/runtime/vulkan/passes/FuseQuantizedLinear.h>
4142
#include <executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
4243
#include <executorch/backends/native/runtime/vulkan/passes/LowerHFAttention.h>
@@ -176,12 +177,13 @@ bool is_functional_in_place(const std::string& target) {
176177
// Ops whose ET-VK implementations require width-packed operands. This list
177178
// seeds layout assignment; propagation below determines the final layouts.
178179
bool is_width_packed_op(std::string_view key) {
179-
constexpr std::array<std::string_view, 11> kOps{
180+
constexpr std::array<std::string_view, 12> kOps{
180181
"aten.linear.default",
181182
"aten.addmm.default",
182183
"aten.mm.default",
183184
"aten.bmm.default",
184185
"et_vk.apply_rotary_emb_hf.default",
186+
"et_vk.embedding_q4gsw.default",
185187
"et_vk.linear_dq8ca_q4gsw.default",
186188
"et_vk.linear_q4gsw.default",
187189
"et_vk.rms_norm.default",
@@ -558,6 +560,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
558560
graph_(std::make_unique<ComputeGraph>(config)) {
559561
method_.graph.rebuild_def_use();
560562
vulkan::lower_hf_attention(method_.graph);
563+
vulkan::fuse_quantized_embeddings(method_);
561564
vulkan::fuse_quantized_linears(method_);
562565
vulkan::lower_rms_norms(method_.graph);
563566
vulkan::insert_prepack_nodes(method_);
Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#include <executorch/backends/native/runtime/vulkan/passes/FuseQuantizedEmbedding.h>
8+
9+
#include <algorithm>
10+
#include <limits>
11+
#include <string_view>
12+
#include <variant>
13+
#include <vector>
14+
15+
#include <executorch/backends/native/runtime/graph/Argument.h>
16+
#include <executorch/backends/native/runtime/graph/Node.h>
17+
#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h>
18+
#include <executorch/backends/native/runtime/vulkan/passes/FuseQuantizedLinear.h>
19+
20+
namespace ptn::vulkan {
21+
namespace {
22+
23+
constexpr std::string_view kEmbedding4Bit =
24+
"torch.ops.quantized_decomposed.embedding_4bit.dtype";
25+
constexpr std::string_view kEmbedding = "torch.ops.aten.embedding.default";
26+
constexpr const char* kEmbeddingQ4gsw =
27+
"torch.ops.et_vk.embedding_q4gsw.default";
28+
29+
const Argument*
30+
get_arg(const Node& node, std::string_view name, size_t position) {
31+
const auto found = std::ranges::find_if(
32+
node.inputs,
33+
[name](const NamedArgument& input) { return input.name == name; });
34+
if (found != node.inputs.end()) {
35+
return &found->arg;
36+
}
37+
return position < node.inputs.size() ? &node.inputs[position].arg : nullptr;
38+
}
39+
40+
const TensorMeta* tensor_meta(const Graph& graph, const Argument* arg) {
41+
if (arg == nullptr || arg->kind() != ArgKind::Tensor ||
42+
!valid(arg->as_tensor().id)) {
43+
return nullptr;
44+
}
45+
const Value& value = graph.value(arg->as_tensor().id);
46+
return value.is_tensor() ? &value.tensor_meta() : nullptr;
47+
}
48+
49+
bool is_int(const Argument* arg, int64_t value) {
50+
return arg != nullptr && arg->kind() == ArgKind::Int &&
51+
!valid(arg->as_int().id) && arg->as_int().value == value;
52+
}
53+
54+
bool is_constant(const Value& value) {
55+
return value.role == ValueRole::Parameter ||
56+
value.role == ValueRole::ConstantTensor;
57+
}
58+
59+
bool is_index_dtype(ScalarType dtype) {
60+
return dtype == ScalarType::Int || dtype == ScalarType::Long;
61+
}
62+
63+
bool rewrite_embedding_4bit(const Graph& graph, Node& node) {
64+
if (!node.is_call() || node.target != kEmbedding4Bit) {
65+
return false;
66+
}
67+
68+
const Argument* weight = get_arg(node, "weight", 0);
69+
const Argument* scales = get_arg(node, "weight_scales", 1);
70+
const Argument* zero_points = get_arg(node, "weight_zero_points", 2);
71+
const Argument* quant_min = get_arg(node, "weight_quant_min", 3);
72+
const Argument* quant_max = get_arg(node, "weight_quant_max", 4);
73+
const Argument* indices = get_arg(node, "indices", 5);
74+
if (weight == nullptr || scales == nullptr || indices == nullptr ||
75+
zero_points == nullptr || zero_points->kind() != ArgKind::None ||
76+
!is_int(quant_min, -8) || !is_int(quant_max, 7)) {
77+
return false;
78+
}
79+
80+
const TensorMeta* weight_meta = tensor_meta(graph, weight);
81+
const TensorMeta* scales_meta = tensor_meta(graph, scales);
82+
const TensorMeta* indices_meta = tensor_meta(graph, indices);
83+
if (weight_meta == nullptr || scales_meta == nullptr ||
84+
indices_meta == nullptr || weight_meta->sizes.size() != 2 ||
85+
scales_meta->sizes.size() != 2 ||
86+
weight_meta->dtype != ScalarType::Byte ||
87+
!is_index_dtype(indices_meta->dtype) ||
88+
weight_meta->sizes[0] != scales_meta->sizes[0] ||
89+
weight_meta->sizes[1] <= 0 || scales_meta->sizes[1] <= 0) {
90+
return false;
91+
}
92+
93+
if (weight_meta->sizes[1] > std::numeric_limits<int64_t>::max() / 2) {
94+
return false;
95+
}
96+
const int64_t embed_dim = weight_meta->sizes[1] * 2;
97+
const int64_t groups_per_row = scales_meta->sizes[1];
98+
if (embed_dim % 32 != 0 || embed_dim % groups_per_row != 0 ||
99+
(embed_dim / groups_per_row) % 4 != 0) {
100+
return false;
101+
}
102+
103+
node.target = kEmbeddingQ4gsw;
104+
node.inputs = {
105+
{.name = "weight", .arg = *weight},
106+
{.name = "weight_scales", .arg = *scales},
107+
{.name = "group_size", .arg = IntArg{embed_dim / groups_per_row}},
108+
{.name = "indices", .arg = *indices},
109+
{.name = "is_linear_weight", .arg = BoolArg{false}},
110+
};
111+
return true;
112+
}
113+
114+
// aten.embedding over a constant carrying AffineGroupQuant, which denotes its
115+
// decoded value. The constant is stored in the q4 linear weight format, so the
116+
// kernel reads it through the linear weight prepack.
117+
bool rewrite_quantized_embedding(Method& method, NodeId node_id) {
118+
Graph& graph = method.graph;
119+
Node& node = graph.node(node_id);
120+
if (!node.is_call() || node.target != kEmbedding) {
121+
return false;
122+
}
123+
const Argument* weight = get_arg(node, "weight", 0);
124+
const Argument* indices = get_arg(node, "indices", 1);
125+
const TensorMeta* weight_meta = tensor_meta(graph, weight);
126+
const TensorMeta* indices_meta = tensor_meta(graph, indices);
127+
if (weight_meta == nullptr || indices_meta == nullptr ||
128+
!weight_meta->quant.has_value() || !is_index_dtype(indices_meta->dtype)) {
129+
return false;
130+
}
131+
const auto* quant = std::get_if<AffineGroupQuant>(&*weight_meta->quant);
132+
const ValueId weight_id = weight->as_tensor().id;
133+
// The transform rewrites the constant for every reader.
134+
if (quant == nullptr || !is_constant(graph.value(weight_id)) ||
135+
graph.value(weight_id).consumer_ids != std::vector<NodeId>{node_id} ||
136+
graph.value(weight_id).attrs.contains(kQ4ConstantTransformAttr) ||
137+
weight_meta->dtype != ScalarType::Byte ||
138+
weight_meta->sizes.size() != 2 || quant->quant_min != -8 ||
139+
quant->quant_max != 7) {
140+
return false;
141+
}
142+
const DataBinding* scales = find_data_binding(method, quant->scale_data_key);
143+
const DataBinding* zeros = quant->zero_point_data_key.empty()
144+
? nullptr
145+
: find_data_binding(method, quant->zero_point_data_key);
146+
if (scales == nullptr ||
147+
(zeros == nullptr && !quant->zero_point_data_key.empty())) {
148+
return false;
149+
}
150+
151+
const int64_t rows = weight_meta->sizes[0];
152+
const int64_t embed_dim = weight_meta->sizes[1];
153+
const int64_t group_size = quant->group_size;
154+
if (rows <= 0 || embed_dim <= 0 || embed_dim % 32 != 0 || group_size <= 0 ||
155+
group_size % 4 != 0 || embed_dim % group_size != 0) {
156+
return false;
157+
}
158+
const Value& scale_value = graph.value(scales->value_id);
159+
if (!is_constant(scale_value) ||
160+
(quant->scale_dtype != ScalarType::Float &&
161+
quant->scale_dtype != ScalarType::Half) ||
162+
scale_value.tensor_meta().dtype != quant->scale_dtype ||
163+
scale_value.tensor_meta().sizes !=
164+
std::vector<int64_t>{rows, embed_dim / group_size} ||
165+
!std::ranges::all_of(scale_value.consumer_ids, [node_id](NodeId id) {
166+
return id == node_id;
167+
})) {
168+
return false;
169+
}
170+
171+
const Argument indices_arg = *indices;
172+
Value& weight_value = graph.value(weight_id);
173+
weight_value.attrs.emplace(
174+
kQ4ConstantTransformAttr,
175+
Q4ConstantTransform{
176+
.kind = Q4ConstantTransformKind::PackWeight,
177+
.source_id = weight_id,
178+
.zero_points_id = zeros == nullptr ? kInvalid : zeros->value_id,
179+
.group_size = group_size,
180+
});
181+
weight_value.tensor_meta() = TensorMeta{
182+
.dtype = ScalarType::Byte,
183+
.sizes = {rows, embed_dim / 2},
184+
};
185+
node.target = kEmbeddingQ4gsw;
186+
node.inputs = {
187+
{.name = "weight", .arg = TensorArg{weight_id}},
188+
{.name = "weight_scales", .arg = TensorArg{scales->value_id}},
189+
{.name = "group_size", .arg = IntArg{group_size}},
190+
{.name = "indices", .arg = indices_arg},
191+
{.name = "is_linear_weight", .arg = BoolArg{true}},
192+
};
193+
return true;
194+
}
195+
196+
} // namespace
197+
198+
// cppcheck-suppress unusedFunction
199+
size_t fuse_quantized_embeddings(Method& method) {
200+
Graph& graph = method.graph;
201+
graph.rebuild_def_use();
202+
const size_t count = static_cast<size_t>(
203+
std::ranges::count_if(graph.schedule, [&](const NodeId node_id) {
204+
return rewrite_embedding_4bit(graph, graph.node(node_id)) ||
205+
rewrite_quantized_embedding(method, node_id);
206+
}));
207+
if (count > 0) {
208+
graph.rebuild_def_use();
209+
validate_graph(graph);
210+
}
211+
return count;
212+
}
213+
214+
} // namespace ptn::vulkan
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#pragma once
8+
9+
#include <cstddef>
10+
11+
#include <executorch/backends/native/runtime/Method.h>
12+
13+
namespace ptn::vulkan {
14+
15+
// Rewrites q4 embeddings into et_vk.embedding_q4gsw: either
16+
// quantized_decomposed.embedding_4bit, or aten.embedding over a constant
17+
// carrying AffineGroupQuant read directly.
18+
size_t fuse_quantized_embeddings(Method& method);
19+
20+
} // namespace ptn::vulkan

‎backends/native/runtime/vulkan/targets.bzl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@ def define_common_targets():
1717
runtime.cxx_library(
1818
name = "vulkan_passes",
1919
srcs = [
20+
"passes/FuseQuantizedEmbedding.cpp",
2021
"passes/FuseQuantizedLinear.cpp",
2122
"passes/InsertPrepack.cpp",
2223
"passes/LowerHFAttention.cpp",
2324
"passes/LowerRMSNorm.cpp",
2425
"passes/MaterializeViewCopies.cpp",
2526
],
2627
exported_headers = [
28+
"passes/FuseQuantizedEmbedding.h",
2729
"passes/FuseQuantizedLinear.h",
2830
"passes/InsertPrepack.h",
2931
"passes/LowerHFAttention.h",

‎backends/native/test/runtime/vulkan/targets.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ def define_common_targets():
4040
runtime.cxx_test(
4141
name = "vulkan_passes_test",
4242
srcs = [
43+
"test_fuse_quantized_embedding.cpp",
4344
"test_fuse_quantized_linear.cpp",
4445
"test_insert_prepack.cpp",
4546
"test_lower_hf_attention.cpp",

0 commit comments

Comments
 (0)