Skip to content

Commit db0d023

Browse files
[executorch][native-vk] Lower Llama operators in Vulkan (pytorch#23461)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23426 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/121/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/121/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/120/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/121/orig Differential Revision: [D119601870](https://our.internmc.facebook.com/intern/diff/D119601870/) @diff-train-skip-merge cc @SS-JIA @manuelcandales @digantdesai @cbilgin Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent 0bdc7c7 commit db0d023

1 file changed

Lines changed: 33 additions & 12 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ std::string registry_key(const std::string& target) {
120120
target.starts_with(kPrefix) ? target.substr(kPrefix.size()) : target;
121121
if (key == "aten.relu_.default") {
122122
key = "aten.relu.default";
123+
} else if (key == "llama.update_cache.default") {
124+
key = "update_cache.default";
123125
}
124126
return key;
125127
}
@@ -133,11 +135,16 @@ bool is_functional_in_place(const std::string& target) {
133135
// Ops whose ET-VK implementations require width-packed operands. This list
134136
// seeds layout assignment; propagation below determines the final layouts.
135137
bool is_width_packed_op(std::string_view key) {
136-
constexpr std::array<std::string_view, 4> kOps{
138+
constexpr std::array<std::string_view, 9> kOps{
137139
"aten.linear.default",
138140
"aten.addmm.default",
139141
"aten.mm.default",
140-
"aten.bmm.default"};
142+
"aten.bmm.default",
143+
"et_vk.linear_dq8ca_q4gsw.default",
144+
"et_vk.rms_norm.default",
145+
"llama.custom_sdpa.default",
146+
"update_cache.default",
147+
"torchao.choose_qparams_affine.default"};
141148
return std::ranges::find(kOps, key) != kOps.end();
142149
}
143150

@@ -898,19 +905,17 @@ class VulkanEngineExecutable final : public EngineExecutable {
898905
throw std::runtime_error("vulkan: unhandled argument kind");
899906
}
900907

901-
// Insert a same-shape view_copy that repacks a channels-packed activation
902-
// into a fresh width-packed tensor, and return the width-packed ref.
903-
VkRef repack_to_width(VkRef src, const Value& v) {
908+
// Insert a same-shape clone into a fresh width-packed tensor when an
909+
// activation's current layout or storage cannot satisfy its consumer.
910+
VkRef repack_to_width(VkRef src, const Value& v, utils::StorageType storage) {
904911
const std::vector<int64_t>& sizes = v.tensor_meta().sizes;
905912
const VkRef dst = graph_->add_tensor(
906913
sizes,
907914
to_vk_dtype(device_dtype(v.tensor_meta().dtype)),
908-
utils::kTexture3D,
915+
storage,
909916
utils::kWidthPacked);
910-
const VkRef size_list =
911-
graph_->add_scalar_list(std::vector<int64_t>(sizes));
912-
std::vector<VkRef> args = {src, size_list, dst};
913-
VK_GET_OP_FN("aten.view_copy.default")(*graph_, args);
917+
std::vector<VkRef> args = {src, none_ref_, dst};
918+
VK_GET_OP_FN("aten.clone.default")(*graph_, args);
914919
return dst;
915920
}
916921

@@ -1022,6 +1027,17 @@ class VulkanEngineExecutable final : public EngineExecutable {
10221027
"vulkan: unsupported op '" + key + "' (target " + n.target + ")");
10231028
}
10241029
const bool width = is_width_packed_op(key);
1030+
std::optional<utils::StorageType> activation_storage;
1031+
if (key == "et_vk.linear_dq8ca_q4gsw.default") {
1032+
const auto output =
1033+
std::ranges::find_if(n.outputs, [this](const Output& candidate) {
1034+
return valid(candidate.value_id) &&
1035+
g().value(candidate.value_id).is_tensor();
1036+
});
1037+
if (output != n.outputs.end()) {
1038+
activation_storage = storage_at(output->value_id);
1039+
}
1040+
}
10251041

10261042
std::vector<VkRef> args;
10271043
args.reserve(n.inputs.size() + 1);
@@ -1038,8 +1054,13 @@ class VulkanEngineExecutable final : public EngineExecutable {
10381054
const Value& v = g().value(value_id);
10391055
const bool activation = v.role == ValueRole::Intermediate ||
10401056
v.role == ValueRole::UserInput;
1041-
if (activation && layout_at(value_id) != utils::kWidthPacked) {
1042-
a = repack_to_width(a, v);
1057+
const bool storage_mismatch = activation_storage.has_value() &&
1058+
na.name == "input" && storage_at(value_id) != *activation_storage;
1059+
if (activation &&
1060+
(layout_at(value_id) != utils::kWidthPacked ||
1061+
storage_mismatch)) {
1062+
a = repack_to_width(
1063+
a, v, activation_storage.value_or(storage_at(value_id)));
10431064
}
10441065
}
10451066
}

0 commit comments

Comments
 (0)