Skip to content

Commit 9bfc99b

Browse files
[executorch][native-vk] Lower Hugging Face static-cache attention at load time (pytorch#23469)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23435 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/129/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/129/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/128/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/129/orig Differential Revision: [D121864327](https://our.internmc.facebook.com/intern/diff/D121864327/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent 19ff441 commit 9bfc99b

6 files changed

Lines changed: 1388 additions & 10 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 142 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <algorithm>
1010
#include <any>
1111
#include <array>
12+
#include <cmath>
1213
#include <cstring>
1314
#include <deque>
1415
#include <iterator>
@@ -37,6 +38,7 @@
3738
#include <executorch/backends/native/runtime/vulkan/VulkanConstantMaterializationTracker.h>
3839
#include <executorch/backends/native/runtime/vulkan/passes/FuseQuantizedLinear.h>
3940
#include <executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
41+
#include <executorch/backends/native/runtime/vulkan/passes/LowerHFAttention.h>
4042
#include <executorch/backends/native/runtime/vulkan/passes/LowerRMSNorm.h>
4143
#include <executorch/backends/native/runtime/vulkan/passes/MaterializeViewCopies.h>
4244
#include <executorch/runtime/core/freeable_buffer.h>
@@ -158,11 +160,12 @@ bool is_functional_in_place(const std::string& target) {
158160
// Ops whose ET-VK implementations require width-packed operands. This list
159161
// seeds layout assignment; propagation below determines the final layouts.
160162
bool is_width_packed_op(std::string_view key) {
161-
constexpr std::array<std::string_view, 9> kOps{
163+
constexpr std::array<std::string_view, 10> kOps{
162164
"aten.linear.default",
163165
"aten.addmm.default",
164166
"aten.mm.default",
165167
"aten.bmm.default",
168+
"et_vk.apply_rotary_emb_hf.default",
166169
"et_vk.linear_dq8ca_q4gsw.default",
167170
"et_vk.rms_norm.default",
168171
"llama.custom_sdpa.default",
@@ -364,6 +367,29 @@ void validate_int64_to_int32(const void* data, size_t numel) {
364367
}
365368
}
366369

370+
void check_cache_position_input(
371+
const void* data,
372+
size_t numel,
373+
ScalarType dtype,
374+
const vulkan::CachePosition& bound,
375+
int64_t& rows_written) {
376+
std::vector<int64_t> positions(numel);
377+
if (dtype == ScalarType::Long) {
378+
std::memcpy(positions.data(), data, numel * sizeof(int64_t));
379+
} else if (dtype == ScalarType::Int) {
380+
const auto* bytes = static_cast<const uint8_t*>(data);
381+
for (size_t i = 0; i < numel; ++i) {
382+
int32_t value;
383+
std::memcpy(&value, bytes + i * sizeof(value), sizeof(value));
384+
positions[i] = value;
385+
}
386+
} else {
387+
throw std::runtime_error(
388+
"vulkan: cache_position input must be int32 or int64");
389+
}
390+
vulkan::check_cache_positions(positions, bound, rows_written);
391+
}
392+
367393
// Holds a method's non-empty mutable DataBinding keys in `owners` for the
368394
// lifetime of its executable.
369395
class MutableStateClaim {
@@ -426,6 +452,12 @@ class VulkanEngineExecutable final : public EngineExecutable {
426452
std::vector<NodeId> nodes_;
427453
std::vector<ValueId> inputs_;
428454
std::vector<ValueId> outputs_;
455+
// Aligned to inputs_: set on the positions the lowered attention reads.
456+
std::vector<std::optional<vulkan::CachePosition>> cache_positions_;
457+
// Rows filled by completed executions. set_input stages its extension in
458+
// pending_cache_rows_written_, which execute() commits once it succeeds.
459+
std::vector<int64_t> cache_rows_written_;
460+
std::vector<int64_t> pending_cache_rows_written_;
429461
// Public output index -> graph output position. Mutation outputs remain in
430462
// outputs_ for execution but are not part of the EngineExecutable API.
431463
std::vector<size_t> user_output_positions_;
@@ -501,6 +533,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
501533
mutable_state_claim_(mutable_state_owners, method),
502534
graph_(std::make_unique<ComputeGraph>(config)) {
503535
method_.graph.rebuild_def_use();
536+
vulkan::lower_hf_attention(method_.graph);
504537
vulkan::fuse_quantized_linears(method_);
505538
vulkan::lower_rms_norms(method_.graph);
506539
vulkan::insert_prepack_nodes(method_);
@@ -509,6 +542,21 @@ class VulkanEngineExecutable final : public EngineExecutable {
509542
nodes_ = method_.graph.schedule;
510543
inputs_ = method_.graph.input_ids;
511544
outputs_ = method_.graph.output_ids;
545+
for (const ValueId input : inputs_) {
546+
const auto& attrs = g().value(input).attrs;
547+
const auto attr = attrs.find(vulkan::kCachePositionAttr);
548+
if (attr == attrs.end()) {
549+
cache_positions_.emplace_back();
550+
continue;
551+
}
552+
const auto* bound = std::any_cast<vulkan::CachePosition>(&attr->second);
553+
if (bound == nullptr) {
554+
throw std::runtime_error("vulkan: malformed cache_position annotation");
555+
}
556+
cache_positions_.emplace_back(*bound);
557+
}
558+
cache_rows_written_.assign(inputs_.size(), 0);
559+
pending_cache_rows_written_ = cache_rows_written_;
512560
if (!method_.output_specs.empty() &&
513561
method_.output_specs.size() != outputs_.size()) {
514562
throw std::runtime_error(
@@ -573,13 +621,19 @@ class VulkanEngineExecutable final : public EngineExecutable {
573621
device_dtype(meta.dtype) == ScalarType::Int) {
574622
validate_int64_to_int32(data, numel);
575623
}
624+
if (const auto& bound = cache_positions_.at(i)) {
625+
int64_t rows_written = cache_rows_written_.at(i);
626+
check_cache_position_input(data, numel, src_dtype, *bound, rows_written);
627+
pending_cache_rows_written_.at(i) = rows_written;
628+
}
576629
graph_->maybe_cast_and_copy_into_staging(
577630
input_staging_.at(i), data, numel, to_vk_dtype(src_dtype));
578631
}
579632

580633
void execute() override {
581634
graph_->propagate_resize();
582635
graph_->execute();
636+
cache_rows_written_ = pending_cache_rows_written_;
583637
}
584638

585639
void get_output(size_t i, void* data, size_t numel, ScalarType dst_dtype)
@@ -867,18 +921,33 @@ class VulkanEngineExecutable final : public EngineExecutable {
867921
return storage;
868922
}
869923

870-
void validate_zero_points(const Package& package, ValueId zero_points_id) {
871-
const DataBinding* binding = binding_at(zero_points_id);
872-
if (binding == nullptr || !binding->has_data) {
924+
void validate_zero_constant(
925+
const Package& package,
926+
ValueId id,
927+
const std::string& what) {
928+
const DataBinding* binding = binding_at(id);
929+
if (binding == nullptr || !binding->has_data || binding->mutated) {
873930
throw std::runtime_error(
874-
"vulkan: q4 weight zero points must be a stored constant");
931+
"vulkan: " + what + " must be a read-only stored constant");
875932
}
876933
const std::shared_ptr<OwnedBytes> storage =
877934
source_constant(package, *binding);
878935
if (!std::ranges::all_of(
879936
storage->span(), [](uint8_t value) { return value == 0; })) {
880-
throw std::runtime_error(
881-
"vulkan: q4 weight zero points must all be zero");
937+
throw std::runtime_error("vulkan: " + what + " must all be zero");
938+
}
939+
}
940+
941+
void validate_zero_points(const Package& package, ValueId zero_points_id) {
942+
validate_zero_constant(package, zero_points_id, "q4 weight zero points");
943+
}
944+
945+
void validate_marked_zero_constants(const Package& package) {
946+
for (ValueId id = 0; id < static_cast<ValueId>(g().values.size()); ++id) {
947+
const Value& value = g().value(id);
948+
if (value.attrs.contains(vulkan::kZeroConstantAttr)) {
949+
validate_zero_constant(package, id, "constant '" + value.name + "'");
950+
}
882951
}
883952
}
884953

@@ -1050,6 +1119,53 @@ class VulkanEngineExecutable final : public EngineExecutable {
10501119
bytes.data(), bytes.size(), release_owned_bytes, context);
10511120
}
10521121

1122+
executorch::runtime::FreeableBuffer derived_rope_table_buffer(
1123+
const Package& package,
1124+
const Value& value,
1125+
const vulkan::RopeTable& table) {
1126+
const DataBinding* source_binding = binding_at(table.inv_freq_id);
1127+
if (source_binding == nullptr || !source_binding->has_data) {
1128+
throw std::runtime_error("vulkan: RoPE table has no stored inv_freq");
1129+
}
1130+
const TensorMeta& meta = value.tensor_meta();
1131+
std::shared_ptr<OwnedBytes>& storage =
1132+
derived_constants_["rope:" + value.name];
1133+
if (!storage) {
1134+
if (meta.dtype != ScalarType::Float) {
1135+
throw std::runtime_error("vulkan: RoPE tables are only built in fp32");
1136+
}
1137+
const std::shared_ptr<OwnedBytes> source =
1138+
source_constant(package, *source_binding);
1139+
const size_t half = source->span().size() / sizeof(float);
1140+
const size_t rows = static_cast<size_t>(meta.sizes.at(0));
1141+
if (static_cast<size_t>(meta.sizes.at(1)) != 2 * half) {
1142+
throw std::runtime_error(
1143+
"vulkan: RoPE table width mismatches inv_freq");
1144+
}
1145+
std::vector<float> inv_freq(half);
1146+
std::memcpy(inv_freq.data(), source->span().data(), half * sizeof(float));
1147+
std::vector<float> values(rows * 2 * half);
1148+
for (size_t p = 0; p < rows; ++p) {
1149+
for (size_t j = 0; j < half; ++j) {
1150+
const float phase = static_cast<float>(p) * inv_freq[j];
1151+
const float v = static_cast<float>(
1152+
(table.use_sin ? std::sin(phase) : std::cos(phase)) *
1153+
table.attention_scale);
1154+
values[p * 2 * half + j] = v;
1155+
values[p * 2 * half + half + j] = v;
1156+
}
1157+
}
1158+
std::vector<uint8_t> bytes(values.size() * sizeof(float));
1159+
std::memcpy(bytes.data(), values.data(), bytes.size());
1160+
storage = std::make_shared<OwnedBytes>(
1161+
OwnedBytes::from_vector(std::move(bytes)));
1162+
}
1163+
const ByteSpan bytes = storage->span();
1164+
auto* context = new std::shared_ptr<OwnedBytes>(storage);
1165+
return executorch::runtime::FreeableBuffer(
1166+
bytes.data(), bytes.size(), release_owned_bytes, context);
1167+
}
1168+
10531169
VkRef make_value(const Package& package, const Value& v, ValueId native_id) {
10541170
switch (v.kind()) {
10551171
case ValueKind::None:
@@ -1077,6 +1193,17 @@ class VulkanEngineExecutable final : public EngineExecutable {
10771193
case ValueKind::Tensor: {
10781194
const TensorMeta& m = v.tensor_meta();
10791195
const std::vector<int64_t>& sizes = m.sizes;
1196+
const auto rope = v.attrs.find(vulkan::kRopeTableAttr);
1197+
if (rope != v.attrs.end()) {
1198+
const auto* table = std::any_cast<vulkan::RopeTable>(&rope->second);
1199+
if (table == nullptr) {
1200+
throw std::runtime_error("vulkan: malformed RoPE table annotation");
1201+
}
1202+
return graph_->add_tensorref(
1203+
sizes,
1204+
to_vk_dtype(device_dtype(m.dtype)),
1205+
derived_rope_table_buffer(package, v, *table));
1206+
}
10801207
const auto transform = v.attrs.find(vulkan::kQ4ConstantTransformAttr);
10811208
if (transform != v.attrs.end()) {
10821209
const auto* q4 =
@@ -1123,7 +1250,8 @@ class VulkanEngineExecutable final : public EngineExecutable {
11231250
VK_GET_OP_FN("et_vk.prepack.default")(*graph_, {source, tensor});
11241251
return tensor;
11251252
}
1126-
if (b != nullptr && b->mutated) {
1253+
if (b != nullptr && b->mutated &&
1254+
!v.attrs.contains(vulkan::kWrittenBeforeReadAttr)) {
11271255
return zero_initialized_tensor(v, native_id);
11281256
}
11291257
return graph_->add_tensor(
@@ -1338,7 +1466,8 @@ class VulkanEngineExecutable final : public EngineExecutable {
13381466
const bool width = is_width_packed_op(key);
13391467
std::optional<utils::StorageType> activation_storage;
13401468
if (key == "aten.linear.default" ||
1341-
key == "et_vk.linear_dq8ca_q4gsw.default") {
1469+
key == "et_vk.linear_dq8ca_q4gsw.default" ||
1470+
key == "et_vk.apply_rotary_emb_hf.default") {
13421471
const auto output =
13431472
std::ranges::find_if(n.outputs, [this](const Output& candidate) {
13441473
return valid(candidate.value_id) &&
@@ -1365,7 +1494,9 @@ class VulkanEngineExecutable final : public EngineExecutable {
13651494
const bool activation = v.role == ValueRole::Intermediate ||
13661495
v.role == ValueRole::UserInput;
13671496
const bool storage_mismatch = activation_storage.has_value() &&
1368-
na.name == "input" && storage_at(value_id) != *activation_storage;
1497+
(key == "et_vk.apply_rotary_emb_hf.default" ||
1498+
na.name == "input") &&
1499+
storage_at(value_id) != *activation_storage;
13691500
if (activation &&
13701501
(layout_at(value_id) != utils::kWidthPacked ||
13711502
storage_mismatch)) {
@@ -1639,6 +1770,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
16391770
detach_in_place_outputs();
16401771
index_output_kinds();
16411772
index_bindings();
1773+
validate_marked_zero_constants(package);
16421774
plan_allocations();
16431775

16441776
// 1. Materialize every native value as an ET-VK value.

0 commit comments

Comments
 (0)