Skip to content

Commit 86d31c4

Browse files
[executorch][native-vk] Use buffer storage for compatible SDPA (pytorch#23476)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23443 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/136/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/136/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/135/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/136/orig Differential Revision: [D121704334](https://our.internmc.facebook.com/intern/diff/D121704334/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent 71edbbc commit 86d31c4

3 files changed

Lines changed: 291 additions & 6 deletions

File tree

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Vulkan engine TODOs
2+
3+
## Plan conversion temporaries in the shared memory plan
4+
5+
Tensors that `VulkanEngineExecutable.cpp` creates for storage or layout
6+
conversions call `graph_->add_tensor` without a `memory_plan_` allocation id,
7+
so each gets dedicated device memory instead of sharing a planned slot:
8+
9+
- `repack_to_width`, including the SDPA query conversion to buffer storage
10+
- the `aten.view_copy.default` storage clone
11+
- the channels-packed `aten.embedding.default` temporary
12+
13+
HuggingFace Llama 3.2 1B 8da4w, 2K export: peak GPU memory is 2794 MiB, against
14+
2265 MiB with buffer SDPA disabled and 2263 MiB for the ET-VK `.pte`. The
15+
+528 MiB comes from enabling buffer SDPA. Give these temporaries lifetimes in
16+
the memory plan, measure again, and attribute whatever remains.

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 72 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,28 @@ bool is_width_packed_op(std::string_view key) {
193193
return std::ranges::find(kOps, key) != kOps.end();
194194
}
195195

196+
// Ops whose activation input is converted to the storage of their output.
197+
bool converts_activation_storage(std::string_view key) {
198+
return key == "aten.linear.default" ||
199+
key == "et_vk.linear_dq8ca_q4gsw.default" ||
200+
key == "et_vk.linear_q4gsw.default" ||
201+
key == "et_vk.apply_rotary_emb_hf.default" ||
202+
key == "llama.custom_sdpa.default";
203+
}
204+
205+
// Which inputs of a converts_activation_storage op take its output storage.
206+
// Only activation tensors are converted: dispatch leaves constants, such as the
207+
// rotary cos and sin tables, in their own storage.
208+
bool is_activation_input(std::string_view key, std::string_view input) {
209+
if (key == "et_vk.apply_rotary_emb_hf.default") {
210+
return true;
211+
}
212+
if (key == "llama.custom_sdpa.default") {
213+
return input == "query";
214+
}
215+
return input == "input";
216+
}
217+
196218
bool preserves_packed_layout(std::string_view key) {
197219
constexpr std::array<std::string_view, 5> kOps{
198220
"aten._to_copy.default",
@@ -723,6 +745,43 @@ class VulkanEngineExecutable final : public EngineExecutable {
723745
return g().value(id).tensor_meta();
724746
}
725747

748+
// ET-VK checks the padded [H, S, C] attention weights against
749+
// max_buffer_numel(), which is a byte limit, and shrinks S instead of
750+
// failing. Only choose buffer when they fit in bytes.
751+
bool buffer_attention_fits(const Node& n) {
752+
const std::string key = registry_key(n.target);
753+
if (key != "llama.custom_sdpa.default") {
754+
return false;
755+
}
756+
const auto tensor_input = [&](std::string_view name) -> const TensorMeta* {
757+
const auto it =
758+
std::ranges::find_if(n.inputs, [&](const NamedArgument& na) {
759+
return na.name == name && na.arg.kind() == ArgKind::Tensor &&
760+
valid(na.arg.as_tensor().id);
761+
});
762+
return it == n.inputs.end()
763+
? nullptr
764+
: &g().value(it->arg.as_tensor().id).tensor_meta();
765+
};
766+
const TensorMeta* query = tensor_input("query");
767+
const TensorMeta* cache = tensor_input("key");
768+
if (query == nullptr || cache == nullptr || query->sizes.size() != 4 ||
769+
cache->sizes.size() != 4) {
770+
return false;
771+
}
772+
const auto align4 = [](int64_t v) { return (v + 3) / 4 * 4; };
773+
const int64_t limit = graph_->max_buffer_numel();
774+
const int64_t elem = static_cast<int64_t>(element_size(query->dtype));
775+
int64_t weights = 0;
776+
if (__builtin_mul_overflow(
777+
query->sizes[2] * align4(query->sizes[1]),
778+
align4(cache->sizes[1]) * elem,
779+
&weights)) {
780+
return false;
781+
}
782+
return weights < limit;
783+
}
784+
726785
void assign_layouts() {
727786
layout_.assign(g().values.size(), utils::kChannelsPacked);
728787
storage_.assign(g().values.size(), utils::kTexture3D);
@@ -791,6 +850,17 @@ class VulkanEngineExecutable final : public EngineExecutable {
791850
});
792851
}
793852

853+
// SDPA stores attention weights in the query's storage. For compatible
854+
// shapes, use buffer storage and convert the query to the output's storage.
855+
for (const NodeId nid : nodes_) {
856+
const Node& n = g().node(nid);
857+
if (n.is_call() && buffer_attention_fits(n)) {
858+
for_each_output_value(n, [&](ValueId output_id) {
859+
storage_at(output_id) = utils::kBuffer;
860+
});
861+
}
862+
}
863+
794864
bool changed;
795865
do {
796866
changed = false;
@@ -1521,10 +1591,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
15211591
}
15221592
const bool width = is_width_packed_op(key);
15231593
std::optional<utils::StorageType> activation_storage;
1524-
if (key == "aten.linear.default" ||
1525-
key == "et_vk.linear_dq8ca_q4gsw.default" ||
1526-
key == "et_vk.linear_q4gsw.default" ||
1527-
key == "et_vk.apply_rotary_emb_hf.default") {
1594+
if (converts_activation_storage(key)) {
15281595
const auto output =
15291596
std::ranges::find_if(n.outputs, [this](const Output& candidate) {
15301597
return valid(candidate.value_id) &&
@@ -1551,8 +1618,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
15511618
const bool activation = v.role == ValueRole::Intermediate ||
15521619
v.role == ValueRole::UserInput;
15531620
const bool storage_mismatch = activation_storage.has_value() &&
1554-
(key == "et_vk.apply_rotary_emb_hf.default" ||
1555-
na.name == "input") &&
1621+
is_activation_input(key, na.name) &&
15561622
storage_at(value_id) != *activation_storage;
15571623
if (activation &&
15581624
(layout_at(value_id) != utils::kWidthPacked ||

‎backends/native/test/runtime/vulkan/test_vulkan_engine.cpp‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include <executorch/backends/native/runtime/vulkan/VulkanEngine.h>
88

9+
#include <cmath>
910
#include <cstdint>
1011
#include <cstring>
1112
#include <limits>
@@ -840,6 +841,148 @@ std::vector<uint8_t> make_dynamic_buffer_view_index_program() {
840841
builder, graph, {fbs::CreateOutputSpecDirect(builder, "gathered")});
841842
}
842843

844+
// Grouped-query causal attention of a query with a dynamic sequence length,
845+
// starting at the position equal to that length, over caches of kSdpaCache.
846+
constexpr int64_t kSdpaSeq = 4;
847+
constexpr int64_t kSdpaCache = 8;
848+
constexpr int64_t kSdpaHeads = 2;
849+
constexpr int64_t kSdpaKvHeads = 1;
850+
constexpr int64_t kSdpaDim = 8;
851+
852+
std::vector<uint8_t> make_dynamic_sdpa_program() {
853+
flatbuffers::FlatBufferBuilder builder;
854+
const auto query_meta = create_tensor_meta(
855+
builder,
856+
fbs::ScalarType::FLOAT,
857+
{1, kSdpaSeq, kSdpaHeads, kSdpaDim},
858+
{1, 1, kSdpaHeads, kSdpaDim});
859+
const auto cache_meta = create_tensor_meta(
860+
builder, fbs::ScalarType::FLOAT, {1, kSdpaCache, kSdpaKvHeads, kSdpaDim});
861+
const auto flat_meta = create_tensor_meta(
862+
builder,
863+
fbs::ScalarType::FLOAT,
864+
{1, kSdpaSeq, kSdpaHeads * kSdpaDim},
865+
{1, 1, kSdpaHeads * kSdpaDim});
866+
const std::vector<flatbuffers::Offset<fbs::TensorValue>> tensor_values = {
867+
fbs::CreateTensorValueDirect(builder, "query", query_meta),
868+
fbs::CreateTensorValueDirect(builder, "key", cache_meta),
869+
fbs::CreateTensorValueDirect(builder, "value", cache_meta),
870+
fbs::CreateTensorValueDirect(builder, "attention", query_meta),
871+
fbs::CreateTensorValueDirect(builder, "flat", flat_meta),
872+
};
873+
874+
const std::vector<flatbuffers::Offset<fbs::Output>> query_outputs = {
875+
fbs::CreateOutputDirect(builder, "query")};
876+
const std::vector<flatbuffers::Offset<fbs::Output>> key_outputs = {
877+
fbs::CreateOutputDirect(builder, "key")};
878+
const std::vector<flatbuffers::Offset<fbs::Output>> value_outputs = {
879+
fbs::CreateOutputDirect(builder, "value")};
880+
const std::vector<flatbuffers::Offset<fbs::Output>> pos_outputs = {
881+
fbs::CreateOutputDirect(
882+
builder, "pos", nullptr, fbs::OutputValueKind::INT)};
883+
const std::vector<flatbuffers::Offset<fbs::Output>> attention_outputs = {
884+
fbs::CreateOutputDirect(builder, "attention")};
885+
const std::vector<flatbuffers::Offset<fbs::Output>> view_outputs = {
886+
fbs::CreateOutputDirect(builder, "flat", "attention")};
887+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> sym_size_inputs = {
888+
fbs::CreateNamedArgumentDirect(
889+
builder, "self", create_tensor_arg(builder, "query")),
890+
fbs::CreateNamedArgumentDirect(
891+
builder, "dim", create_int_arg(builder, 1)),
892+
};
893+
const auto start_pos = fbs::CreateIntArgDirect(builder, 0, "pos");
894+
const auto dropout = fbs::CreateFloatArg(builder, 0.0);
895+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> sdpa_inputs = {
896+
fbs::CreateNamedArgumentDirect(
897+
builder, "query", create_tensor_arg(builder, "query")),
898+
fbs::CreateNamedArgumentDirect(
899+
builder, "key", create_tensor_arg(builder, "key")),
900+
fbs::CreateNamedArgumentDirect(
901+
builder, "value", create_tensor_arg(builder, "value")),
902+
fbs::CreateNamedArgumentDirect(
903+
builder,
904+
"start_pos",
905+
fbs::CreateArgument(
906+
builder, fbs::ArgumentValue::IntArg, start_pos.Union())),
907+
fbs::CreateNamedArgumentDirect(
908+
builder, "attn_mask", create_none_arg(builder)),
909+
fbs::CreateNamedArgumentDirect(
910+
builder,
911+
"drpout_p",
912+
fbs::CreateArgument(
913+
builder, fbs::ArgumentValue::FloatArg, dropout.Union())),
914+
fbs::CreateNamedArgumentDirect(
915+
builder, "is_causal", create_bool_arg(builder, true)),
916+
fbs::CreateNamedArgumentDirect(
917+
builder, "scale", create_none_arg(builder)),
918+
};
919+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> view_inputs = {
920+
fbs::CreateNamedArgumentDirect(
921+
builder, "self", create_tensor_arg(builder, "attention")),
922+
fbs::CreateNamedArgumentDirect(
923+
builder,
924+
"size",
925+
create_int_list_arg(builder, {1, -1, kSdpaHeads * kSdpaDim})),
926+
};
927+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> output_inputs = {
928+
fbs::CreateNamedArgumentDirect(
929+
builder, "", create_tensor_arg(builder, "flat"))};
930+
const std::vector<flatbuffers::Offset<fbs::Node>> nodes = {
931+
fbs::CreateNodeDirect(
932+
builder,
933+
"query",
934+
fbs::OpKind::PLACEHOLDER,
935+
"",
936+
nullptr,
937+
&query_outputs),
938+
fbs::CreateNodeDirect(
939+
builder, "key", fbs::OpKind::PLACEHOLDER, "", nullptr, &key_outputs),
940+
fbs::CreateNodeDirect(
941+
builder,
942+
"value",
943+
fbs::OpKind::PLACEHOLDER,
944+
"",
945+
nullptr,
946+
&value_outputs),
947+
fbs::CreateNodeDirect(
948+
builder,
949+
"pos",
950+
fbs::OpKind::CALL_FUNCTION,
951+
"torch.ops.aten.sym_size.int",
952+
&sym_size_inputs,
953+
&pos_outputs),
954+
fbs::CreateNodeDirect(
955+
builder,
956+
"sdpa",
957+
fbs::OpKind::CALL_FUNCTION,
958+
"torch.ops.llama.custom_sdpa.default",
959+
&sdpa_inputs,
960+
&attention_outputs),
961+
fbs::CreateNodeDirect(
962+
builder,
963+
"view",
964+
fbs::OpKind::CALL_FUNCTION,
965+
"torch.ops.aten.view.default",
966+
&view_inputs,
967+
&view_outputs),
968+
fbs::CreateNodeDirect(
969+
builder, "output", fbs::OpKind::OUTPUT, "", &output_inputs),
970+
};
971+
const auto graph = fbs::CreateGraph(
972+
builder,
973+
builder.CreateVector(nodes),
974+
create_strings(builder, {"query", "key", "value"}),
975+
create_strings(builder, {"flat"}),
976+
builder.CreateVector(tensor_values));
977+
return finish_program(
978+
builder, graph, {fbs::CreateOutputSpecDirect(builder, "flat")});
979+
}
980+
981+
float sdpa_input_value(size_t index, int64_t salt) {
982+
return 0.125f * static_cast<float>((static_cast<int64_t>(index) * salt) % 9) -
983+
0.5f;
984+
}
985+
843986
// Values of the int4 [4, 8] weight in groups of 4 used by the program below.
844987
constexpr int64_t kQ4Out = 4;
845988
constexpr int64_t kQ4In = 8;
@@ -1381,6 +1524,66 @@ TEST(VulkanEngineTest, RejectsBufferViewResizeThatDoesNotDivide) {
13811524
EXPECT_THROW(compiled.executable->execute(), std::runtime_error);
13821525
}
13831526

1527+
TEST(VulkanEngineTest, ExecutesCausalSdpaOnResizedQuery) {
1528+
CompiledProgram compiled = compile_program(make_dynamic_sdpa_program());
1529+
constexpr int64_t kSeq = 3;
1530+
compiled.executable->resize_input(0, {1, kSeq, kSdpaHeads, kSdpaDim});
1531+
1532+
std::vector<float> query(kSeq * kSdpaHeads * kSdpaDim);
1533+
std::vector<float> key(kSdpaCache * kSdpaKvHeads * kSdpaDim);
1534+
std::vector<float> value(key.size());
1535+
for (size_t i = 0; i < query.size(); ++i) {
1536+
query[i] = sdpa_input_value(i, 5);
1537+
}
1538+
for (size_t i = 0; i < key.size(); ++i) {
1539+
key[i] = sdpa_input_value(i, 7);
1540+
value[i] = sdpa_input_value(i, 4);
1541+
}
1542+
compiled.executable->set_input(
1543+
0, query.data(), query.size(), ScalarType::Float);
1544+
compiled.executable->set_input(1, key.data(), key.size(), ScalarType::Float);
1545+
compiled.executable->set_input(
1546+
2, value.data(), value.size(), ScalarType::Float);
1547+
compiled.executable->execute();
1548+
1549+
EXPECT_EQ(
1550+
compiled.executable->output_sizes(0),
1551+
(std::vector<int64_t>{1, kSeq, kSdpaHeads * kSdpaDim}));
1552+
std::vector<float> output(query.size());
1553+
compiled.executable->get_output(
1554+
0, output.data(), output.size(), ScalarType::Float);
1555+
1556+
const float scale = 1.0f / std::sqrt(static_cast<float>(kSdpaDim));
1557+
for (int64_t s = 0; s < kSeq; ++s) {
1558+
const int64_t context = kSeq + s + 1;
1559+
for (int64_t h = 0; h < kSdpaHeads; ++h) {
1560+
const int64_t kv_head = h / (kSdpaHeads / kSdpaKvHeads);
1561+
const float* q = &query[(s * kSdpaHeads + h) * kSdpaDim];
1562+
std::vector<float> weights(context);
1563+
float total = 0.0f;
1564+
for (int64_t c = 0; c < context; ++c) {
1565+
const float* k = &key[(c * kSdpaKvHeads + kv_head) * kSdpaDim];
1566+
float dot = 0.0f;
1567+
for (int64_t d = 0; d < kSdpaDim; ++d) {
1568+
dot += q[d] * k[d];
1569+
}
1570+
weights[c] = std::exp(dot * scale);
1571+
total += weights[c];
1572+
}
1573+
for (int64_t d = 0; d < kSdpaDim; ++d) {
1574+
float expected = 0.0f;
1575+
for (int64_t c = 0; c < context; ++c) {
1576+
expected += weights[c] / total *
1577+
value[(c * kSdpaKvHeads + kv_head) * kSdpaDim + d];
1578+
}
1579+
EXPECT_NEAR(
1580+
output[(s * kSdpaHeads + h) * kSdpaDim + d], expected, 1e-4f)
1581+
<< "position " << s << " head " << h << " dim " << d;
1582+
}
1583+
}
1584+
}
1585+
}
1586+
13841587
TEST(VulkanEngineTest, ExecutesLinearOnDirectlyQuantizedWeight) {
13851588
CompiledProgram compiled = compile_program(
13861589
make_direct_q4_linear_program(), make_direct_q4_linear_tensors());

0 commit comments

Comments
 (0)