|
| 1 | +// Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +// All rights reserved. |
| 3 | +// |
| 4 | +// This source code is licensed under the BSD-style license found in the |
| 5 | +// LICENSE file in the root directory of this source tree. |
| 6 | + |
| 7 | +#include <executorch/backends/native/runtime/vulkan/passes/FuseQuantizedEmbedding.h> |
| 8 | + |
| 9 | +#include <algorithm> |
| 10 | +#include <limits> |
| 11 | +#include <string_view> |
| 12 | +#include <variant> |
| 13 | +#include <vector> |
| 14 | + |
| 15 | +#include <executorch/backends/native/runtime/graph/Argument.h> |
| 16 | +#include <executorch/backends/native/runtime/graph/Node.h> |
| 17 | +#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h> |
| 18 | +#include <executorch/backends/native/runtime/vulkan/passes/FuseQuantizedLinear.h> |
| 19 | + |
| 20 | +namespace ptn::vulkan { |
| 21 | +namespace { |
| 22 | + |
| 23 | +constexpr std::string_view kEmbedding4Bit = |
| 24 | + "torch.ops.quantized_decomposed.embedding_4bit.dtype"; |
| 25 | +constexpr std::string_view kEmbedding = "torch.ops.aten.embedding.default"; |
| 26 | +constexpr const char* kEmbeddingQ4gsw = |
| 27 | + "torch.ops.et_vk.embedding_q4gsw.default"; |
| 28 | + |
| 29 | +const Argument* |
| 30 | +get_arg(const Node& node, std::string_view name, size_t position) { |
| 31 | + const auto found = std::ranges::find_if( |
| 32 | + node.inputs, |
| 33 | + [name](const NamedArgument& input) { return input.name == name; }); |
| 34 | + if (found != node.inputs.end()) { |
| 35 | + return &found->arg; |
| 36 | + } |
| 37 | + return position < node.inputs.size() ? &node.inputs[position].arg : nullptr; |
| 38 | +} |
| 39 | + |
| 40 | +const TensorMeta* tensor_meta(const Graph& graph, const Argument* arg) { |
| 41 | + if (arg == nullptr || arg->kind() != ArgKind::Tensor || |
| 42 | + !valid(arg->as_tensor().id)) { |
| 43 | + return nullptr; |
| 44 | + } |
| 45 | + const Value& value = graph.value(arg->as_tensor().id); |
| 46 | + return value.is_tensor() ? &value.tensor_meta() : nullptr; |
| 47 | +} |
| 48 | + |
| 49 | +bool is_int(const Argument* arg, int64_t value) { |
| 50 | + return arg != nullptr && arg->kind() == ArgKind::Int && |
| 51 | + !valid(arg->as_int().id) && arg->as_int().value == value; |
| 52 | +} |
| 53 | + |
| 54 | +bool is_constant(const Value& value) { |
| 55 | + return value.role == ValueRole::Parameter || |
| 56 | + value.role == ValueRole::ConstantTensor; |
| 57 | +} |
| 58 | + |
| 59 | +bool is_index_dtype(ScalarType dtype) { |
| 60 | + return dtype == ScalarType::Int || dtype == ScalarType::Long; |
| 61 | +} |
| 62 | + |
| 63 | +bool rewrite_embedding_4bit(const Graph& graph, Node& node) { |
| 64 | + if (!node.is_call() || node.target != kEmbedding4Bit) { |
| 65 | + return false; |
| 66 | + } |
| 67 | + |
| 68 | + const Argument* weight = get_arg(node, "weight", 0); |
| 69 | + const Argument* scales = get_arg(node, "weight_scales", 1); |
| 70 | + const Argument* zero_points = get_arg(node, "weight_zero_points", 2); |
| 71 | + const Argument* quant_min = get_arg(node, "weight_quant_min", 3); |
| 72 | + const Argument* quant_max = get_arg(node, "weight_quant_max", 4); |
| 73 | + const Argument* indices = get_arg(node, "indices", 5); |
| 74 | + if (weight == nullptr || scales == nullptr || indices == nullptr || |
| 75 | + zero_points == nullptr || zero_points->kind() != ArgKind::None || |
| 76 | + !is_int(quant_min, -8) || !is_int(quant_max, 7)) { |
| 77 | + return false; |
| 78 | + } |
| 79 | + |
| 80 | + const TensorMeta* weight_meta = tensor_meta(graph, weight); |
| 81 | + const TensorMeta* scales_meta = tensor_meta(graph, scales); |
| 82 | + const TensorMeta* indices_meta = tensor_meta(graph, indices); |
| 83 | + if (weight_meta == nullptr || scales_meta == nullptr || |
| 84 | + indices_meta == nullptr || weight_meta->sizes.size() != 2 || |
| 85 | + scales_meta->sizes.size() != 2 || |
| 86 | + weight_meta->dtype != ScalarType::Byte || |
| 87 | + !is_index_dtype(indices_meta->dtype) || |
| 88 | + weight_meta->sizes[0] != scales_meta->sizes[0] || |
| 89 | + weight_meta->sizes[1] <= 0 || scales_meta->sizes[1] <= 0) { |
| 90 | + return false; |
| 91 | + } |
| 92 | + |
| 93 | + if (weight_meta->sizes[1] > std::numeric_limits<int64_t>::max() / 2) { |
| 94 | + return false; |
| 95 | + } |
| 96 | + const int64_t embed_dim = weight_meta->sizes[1] * 2; |
| 97 | + const int64_t groups_per_row = scales_meta->sizes[1]; |
| 98 | + if (embed_dim % 32 != 0 || embed_dim % groups_per_row != 0 || |
| 99 | + (embed_dim / groups_per_row) % 4 != 0) { |
| 100 | + return false; |
| 101 | + } |
| 102 | + |
| 103 | + node.target = kEmbeddingQ4gsw; |
| 104 | + node.inputs = { |
| 105 | + {.name = "weight", .arg = *weight}, |
| 106 | + {.name = "weight_scales", .arg = *scales}, |
| 107 | + {.name = "group_size", .arg = IntArg{embed_dim / groups_per_row}}, |
| 108 | + {.name = "indices", .arg = *indices}, |
| 109 | + {.name = "is_linear_weight", .arg = BoolArg{false}}, |
| 110 | + }; |
| 111 | + return true; |
| 112 | +} |
| 113 | + |
| 114 | +// aten.embedding over a constant carrying AffineGroupQuant, which denotes its |
| 115 | +// decoded value. The constant is stored in the q4 linear weight format, so the |
| 116 | +// kernel reads it through the linear weight prepack. |
| 117 | +bool rewrite_quantized_embedding(Method& method, NodeId node_id) { |
| 118 | + Graph& graph = method.graph; |
| 119 | + Node& node = graph.node(node_id); |
| 120 | + if (!node.is_call() || node.target != kEmbedding) { |
| 121 | + return false; |
| 122 | + } |
| 123 | + const Argument* weight = get_arg(node, "weight", 0); |
| 124 | + const Argument* indices = get_arg(node, "indices", 1); |
| 125 | + const TensorMeta* weight_meta = tensor_meta(graph, weight); |
| 126 | + const TensorMeta* indices_meta = tensor_meta(graph, indices); |
| 127 | + if (weight_meta == nullptr || indices_meta == nullptr || |
| 128 | + !weight_meta->quant.has_value() || !is_index_dtype(indices_meta->dtype)) { |
| 129 | + return false; |
| 130 | + } |
| 131 | + const auto* quant = std::get_if<AffineGroupQuant>(&*weight_meta->quant); |
| 132 | + const ValueId weight_id = weight->as_tensor().id; |
| 133 | + // The transform rewrites the constant for every reader. |
| 134 | + if (quant == nullptr || !is_constant(graph.value(weight_id)) || |
| 135 | + graph.value(weight_id).consumer_ids != std::vector<NodeId>{node_id} || |
| 136 | + graph.value(weight_id).attrs.contains(kQ4ConstantTransformAttr) || |
| 137 | + weight_meta->dtype != ScalarType::Byte || |
| 138 | + weight_meta->sizes.size() != 2 || quant->quant_min != -8 || |
| 139 | + quant->quant_max != 7) { |
| 140 | + return false; |
| 141 | + } |
| 142 | + const DataBinding* scales = find_data_binding(method, quant->scale_data_key); |
| 143 | + const DataBinding* zeros = quant->zero_point_data_key.empty() |
| 144 | + ? nullptr |
| 145 | + : find_data_binding(method, quant->zero_point_data_key); |
| 146 | + if (scales == nullptr || |
| 147 | + (zeros == nullptr && !quant->zero_point_data_key.empty())) { |
| 148 | + return false; |
| 149 | + } |
| 150 | + |
| 151 | + const int64_t rows = weight_meta->sizes[0]; |
| 152 | + const int64_t embed_dim = weight_meta->sizes[1]; |
| 153 | + const int64_t group_size = quant->group_size; |
| 154 | + if (rows <= 0 || embed_dim <= 0 || embed_dim % 32 != 0 || group_size <= 0 || |
| 155 | + group_size % 4 != 0 || embed_dim % group_size != 0) { |
| 156 | + return false; |
| 157 | + } |
| 158 | + const Value& scale_value = graph.value(scales->value_id); |
| 159 | + if (!is_constant(scale_value) || |
| 160 | + (quant->scale_dtype != ScalarType::Float && |
| 161 | + quant->scale_dtype != ScalarType::Half) || |
| 162 | + scale_value.tensor_meta().dtype != quant->scale_dtype || |
| 163 | + scale_value.tensor_meta().sizes != |
| 164 | + std::vector<int64_t>{rows, embed_dim / group_size} || |
| 165 | + !std::ranges::all_of(scale_value.consumer_ids, [node_id](NodeId id) { |
| 166 | + return id == node_id; |
| 167 | + })) { |
| 168 | + return false; |
| 169 | + } |
| 170 | + |
| 171 | + const Argument indices_arg = *indices; |
| 172 | + Value& weight_value = graph.value(weight_id); |
| 173 | + weight_value.attrs.emplace( |
| 174 | + kQ4ConstantTransformAttr, |
| 175 | + Q4ConstantTransform{ |
| 176 | + .kind = Q4ConstantTransformKind::PackWeight, |
| 177 | + .source_id = weight_id, |
| 178 | + .zero_points_id = zeros == nullptr ? kInvalid : zeros->value_id, |
| 179 | + .group_size = group_size, |
| 180 | + }); |
| 181 | + weight_value.tensor_meta() = TensorMeta{ |
| 182 | + .dtype = ScalarType::Byte, |
| 183 | + .sizes = {rows, embed_dim / 2}, |
| 184 | + }; |
| 185 | + node.target = kEmbeddingQ4gsw; |
| 186 | + node.inputs = { |
| 187 | + {.name = "weight", .arg = TensorArg{weight_id}}, |
| 188 | + {.name = "weight_scales", .arg = TensorArg{scales->value_id}}, |
| 189 | + {.name = "group_size", .arg = IntArg{group_size}}, |
| 190 | + {.name = "indices", .arg = indices_arg}, |
| 191 | + {.name = "is_linear_weight", .arg = BoolArg{true}}, |
| 192 | + }; |
| 193 | + return true; |
| 194 | +} |
| 195 | + |
| 196 | +} // namespace |
| 197 | + |
| 198 | +// cppcheck-suppress unusedFunction |
| 199 | +size_t fuse_quantized_embeddings(Method& method) { |
| 200 | + Graph& graph = method.graph; |
| 201 | + graph.rebuild_def_use(); |
| 202 | + const size_t count = static_cast<size_t>( |
| 203 | + std::ranges::count_if(graph.schedule, [&](const NodeId node_id) { |
| 204 | + return rewrite_embedding_4bit(graph, graph.node(node_id)) || |
| 205 | + rewrite_quantized_embedding(method, node_id); |
| 206 | + })); |
| 207 | + if (count > 0) { |
| 208 | + graph.rebuild_def_use(); |
| 209 | + validate_graph(graph); |
| 210 | + } |
| 211 | + return count; |
| 212 | +} |
| 213 | + |
| 214 | +} // namespace ptn::vulkan |
0 commit comments