99#include < algorithm>
1010#include < any>
1111#include < array>
12+ #include < cmath>
1213#include < cstring>
1314#include < deque>
1415#include < iterator>
3738#include < executorch/backends/native/runtime/vulkan/VulkanConstantMaterializationTracker.h>
3839#include < executorch/backends/native/runtime/vulkan/passes/FuseQuantizedLinear.h>
3940#include < executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
41+ #include < executorch/backends/native/runtime/vulkan/passes/LowerHFAttention.h>
4042#include < executorch/backends/native/runtime/vulkan/passes/LowerRMSNorm.h>
4143#include < executorch/backends/native/runtime/vulkan/passes/MaterializeViewCopies.h>
4244#include < executorch/runtime/core/freeable_buffer.h>
@@ -158,11 +160,12 @@ bool is_functional_in_place(const std::string& target) {
158160// Ops whose ET-VK implementations require width-packed operands. This list
159161// seeds layout assignment; propagation below determines the final layouts.
160162bool is_width_packed_op (std::string_view key) {
161- constexpr std::array<std::string_view, 9 > kOps {
163+ constexpr std::array<std::string_view, 10 > kOps {
162164 " aten.linear.default" ,
163165 " aten.addmm.default" ,
164166 " aten.mm.default" ,
165167 " aten.bmm.default" ,
168+ " et_vk.apply_rotary_emb_hf.default" ,
166169 " et_vk.linear_dq8ca_q4gsw.default" ,
167170 " et_vk.rms_norm.default" ,
168171 " llama.custom_sdpa.default" ,
@@ -364,6 +367,29 @@ void validate_int64_to_int32(const void* data, size_t numel) {
364367 }
365368}
366369
370+ void check_cache_position_input (
371+ const void * data,
372+ size_t numel,
373+ ScalarType dtype,
374+ const vulkan::CachePosition& bound,
375+ int64_t & rows_written) {
376+ std::vector<int64_t > positions (numel);
377+ if (dtype == ScalarType::Long) {
378+ std::memcpy (positions.data (), data, numel * sizeof (int64_t ));
379+ } else if (dtype == ScalarType::Int) {
380+ const auto * bytes = static_cast <const uint8_t *>(data);
381+ for (size_t i = 0 ; i < numel; ++i) {
382+ int32_t value;
383+ std::memcpy (&value, bytes + i * sizeof (value), sizeof (value));
384+ positions[i] = value;
385+ }
386+ } else {
387+ throw std::runtime_error (
388+ " vulkan: cache_position input must be int32 or int64" );
389+ }
390+ vulkan::check_cache_positions (positions, bound, rows_written);
391+ }
392+
367393// Holds a method's non-empty mutable DataBinding keys in `owners` for the
368394// lifetime of its executable.
369395class MutableStateClaim {
@@ -426,6 +452,12 @@ class VulkanEngineExecutable final : public EngineExecutable {
426452 std::vector<NodeId> nodes_;
427453 std::vector<ValueId> inputs_;
428454 std::vector<ValueId> outputs_;
455+ // Aligned to inputs_: set on the positions the lowered attention reads.
456+ std::vector<std::optional<vulkan::CachePosition>> cache_positions_;
457+ // Rows filled by completed executions. set_input stages its extension in
458+ // pending_cache_rows_written_, which execute() commits once it succeeds.
459+ std::vector<int64_t > cache_rows_written_;
460+ std::vector<int64_t > pending_cache_rows_written_;
429461 // Public output index -> graph output position. Mutation outputs remain in
430462 // outputs_ for execution but are not part of the EngineExecutable API.
431463 std::vector<size_t > user_output_positions_;
@@ -501,6 +533,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
501533 mutable_state_claim_(mutable_state_owners, method),
502534 graph_(std::make_unique<ComputeGraph>(config)) {
503535 method_.graph .rebuild_def_use ();
536+ vulkan::lower_hf_attention (method_.graph );
504537 vulkan::fuse_quantized_linears (method_);
505538 vulkan::lower_rms_norms (method_.graph );
506539 vulkan::insert_prepack_nodes (method_);
@@ -509,6 +542,21 @@ class VulkanEngineExecutable final : public EngineExecutable {
509542 nodes_ = method_.graph .schedule ;
510543 inputs_ = method_.graph .input_ids ;
511544 outputs_ = method_.graph .output_ids ;
545+ for (const ValueId input : inputs_) {
546+ const auto & attrs = g ().value (input).attrs ;
547+ const auto attr = attrs.find (vulkan::kCachePositionAttr );
548+ if (attr == attrs.end ()) {
549+ cache_positions_.emplace_back ();
550+ continue ;
551+ }
552+ const auto * bound = std::any_cast<vulkan::CachePosition>(&attr->second );
553+ if (bound == nullptr ) {
554+ throw std::runtime_error (" vulkan: malformed cache_position annotation" );
555+ }
556+ cache_positions_.emplace_back (*bound);
557+ }
558+ cache_rows_written_.assign (inputs_.size (), 0 );
559+ pending_cache_rows_written_ = cache_rows_written_;
512560 if (!method_.output_specs .empty () &&
513561 method_.output_specs .size () != outputs_.size ()) {
514562 throw std::runtime_error (
@@ -573,13 +621,19 @@ class VulkanEngineExecutable final : public EngineExecutable {
573621 device_dtype (meta.dtype ) == ScalarType::Int) {
574622 validate_int64_to_int32 (data, numel);
575623 }
624+ if (const auto & bound = cache_positions_.at (i)) {
625+ int64_t rows_written = cache_rows_written_.at (i);
626+ check_cache_position_input (data, numel, src_dtype, *bound, rows_written);
627+ pending_cache_rows_written_.at (i) = rows_written;
628+ }
576629 graph_->maybe_cast_and_copy_into_staging (
577630 input_staging_.at (i), data, numel, to_vk_dtype (src_dtype));
578631 }
579632
580633 void execute () override {
581634 graph_->propagate_resize ();
582635 graph_->execute ();
636+ cache_rows_written_ = pending_cache_rows_written_;
583637 }
584638
585639 void get_output (size_t i, void * data, size_t numel, ScalarType dst_dtype)
@@ -867,18 +921,33 @@ class VulkanEngineExecutable final : public EngineExecutable {
867921 return storage;
868922 }
869923
870- void validate_zero_points (const Package& package, ValueId zero_points_id) {
871- const DataBinding* binding = binding_at (zero_points_id);
872- if (binding == nullptr || !binding->has_data ) {
924+ void validate_zero_constant (
925+ const Package& package,
926+ ValueId id,
927+ const std::string& what) {
928+ const DataBinding* binding = binding_at (id);
929+ if (binding == nullptr || !binding->has_data || binding->mutated ) {
873930 throw std::runtime_error (
874- " vulkan: q4 weight zero points must be a stored constant" );
931+ " vulkan: " + what + " must be a read-only stored constant" );
875932 }
876933 const std::shared_ptr<OwnedBytes> storage =
877934 source_constant (package, *binding);
878935 if (!std::ranges::all_of (
879936 storage->span (), [](uint8_t value) { return value == 0 ; })) {
880- throw std::runtime_error (
881- " vulkan: q4 weight zero points must all be zero" );
937+ throw std::runtime_error (" vulkan: " + what + " must all be zero" );
938+ }
939+ }
940+
941+ void validate_zero_points (const Package& package, ValueId zero_points_id) {
942+ validate_zero_constant (package, zero_points_id, " q4 weight zero points" );
943+ }
944+
945+ void validate_marked_zero_constants (const Package& package) {
946+ for (ValueId id = 0 ; id < static_cast <ValueId>(g ().values .size ()); ++id) {
947+ const Value& value = g ().value (id);
948+ if (value.attrs .contains (vulkan::kZeroConstantAttr )) {
949+ validate_zero_constant (package, id, " constant '" + value.name + " '" );
950+ }
882951 }
883952 }
884953
@@ -1050,6 +1119,53 @@ class VulkanEngineExecutable final : public EngineExecutable {
10501119 bytes.data (), bytes.size (), release_owned_bytes, context);
10511120 }
10521121
1122+ executorch::runtime::FreeableBuffer derived_rope_table_buffer (
1123+ const Package& package,
1124+ const Value& value,
1125+ const vulkan::RopeTable& table) {
1126+ const DataBinding* source_binding = binding_at (table.inv_freq_id );
1127+ if (source_binding == nullptr || !source_binding->has_data ) {
1128+ throw std::runtime_error (" vulkan: RoPE table has no stored inv_freq" );
1129+ }
1130+ const TensorMeta& meta = value.tensor_meta ();
1131+ std::shared_ptr<OwnedBytes>& storage =
1132+ derived_constants_[" rope:" + value.name ];
1133+ if (!storage) {
1134+ if (meta.dtype != ScalarType::Float) {
1135+ throw std::runtime_error (" vulkan: RoPE tables are only built in fp32" );
1136+ }
1137+ const std::shared_ptr<OwnedBytes> source =
1138+ source_constant (package, *source_binding);
1139+ const size_t half = source->span ().size () / sizeof (float );
1140+ const size_t rows = static_cast <size_t >(meta.sizes .at (0 ));
1141+ if (static_cast <size_t >(meta.sizes .at (1 )) != 2 * half) {
1142+ throw std::runtime_error (
1143+ " vulkan: RoPE table width mismatches inv_freq" );
1144+ }
1145+ std::vector<float > inv_freq (half);
1146+ std::memcpy (inv_freq.data (), source->span ().data (), half * sizeof (float ));
1147+ std::vector<float > values (rows * 2 * half);
1148+ for (size_t p = 0 ; p < rows; ++p) {
1149+ for (size_t j = 0 ; j < half; ++j) {
1150+ const float phase = static_cast <float >(p) * inv_freq[j];
1151+ const float v = static_cast <float >(
1152+ (table.use_sin ? std::sin (phase) : std::cos (phase)) *
1153+ table.attention_scale );
1154+ values[p * 2 * half + j] = v;
1155+ values[p * 2 * half + half + j] = v;
1156+ }
1157+ }
1158+ std::vector<uint8_t > bytes (values.size () * sizeof (float ));
1159+ std::memcpy (bytes.data (), values.data (), bytes.size ());
1160+ storage = std::make_shared<OwnedBytes>(
1161+ OwnedBytes::from_vector (std::move (bytes)));
1162+ }
1163+ const ByteSpan bytes = storage->span ();
1164+ auto * context = new std::shared_ptr<OwnedBytes>(storage);
1165+ return executorch::runtime::FreeableBuffer (
1166+ bytes.data (), bytes.size (), release_owned_bytes, context);
1167+ }
1168+
10531169 VkRef make_value (const Package& package, const Value& v, ValueId native_id) {
10541170 switch (v.kind ()) {
10551171 case ValueKind::None:
@@ -1077,6 +1193,17 @@ class VulkanEngineExecutable final : public EngineExecutable {
10771193 case ValueKind::Tensor: {
10781194 const TensorMeta& m = v.tensor_meta ();
10791195 const std::vector<int64_t >& sizes = m.sizes ;
1196+ const auto rope = v.attrs .find (vulkan::kRopeTableAttr );
1197+ if (rope != v.attrs .end ()) {
1198+ const auto * table = std::any_cast<vulkan::RopeTable>(&rope->second );
1199+ if (table == nullptr ) {
1200+ throw std::runtime_error (" vulkan: malformed RoPE table annotation" );
1201+ }
1202+ return graph_->add_tensorref (
1203+ sizes,
1204+ to_vk_dtype (device_dtype (m.dtype )),
1205+ derived_rope_table_buffer (package, v, *table));
1206+ }
10801207 const auto transform = v.attrs .find (vulkan::kQ4ConstantTransformAttr );
10811208 if (transform != v.attrs .end ()) {
10821209 const auto * q4 =
@@ -1123,7 +1250,8 @@ class VulkanEngineExecutable final : public EngineExecutable {
11231250 VK_GET_OP_FN (" et_vk.prepack.default" )(*graph_, {source, tensor});
11241251 return tensor;
11251252 }
1126- if (b != nullptr && b->mutated ) {
1253+ if (b != nullptr && b->mutated &&
1254+ !v.attrs .contains (vulkan::kWrittenBeforeReadAttr )) {
11271255 return zero_initialized_tensor (v, native_id);
11281256 }
11291257 return graph_->add_tensor (
@@ -1338,7 +1466,8 @@ class VulkanEngineExecutable final : public EngineExecutable {
13381466 const bool width = is_width_packed_op (key);
13391467 std::optional<utils::StorageType> activation_storage;
13401468 if (key == " aten.linear.default" ||
1341- key == " et_vk.linear_dq8ca_q4gsw.default" ) {
1469+ key == " et_vk.linear_dq8ca_q4gsw.default" ||
1470+ key == " et_vk.apply_rotary_emb_hf.default" ) {
13421471 const auto output =
13431472 std::ranges::find_if (n.outputs , [this ](const Output& candidate) {
13441473 return valid (candidate.value_id ) &&
@@ -1365,7 +1494,9 @@ class VulkanEngineExecutable final : public EngineExecutable {
13651494 const bool activation = v.role == ValueRole::Intermediate ||
13661495 v.role == ValueRole::UserInput;
13671496 const bool storage_mismatch = activation_storage.has_value () &&
1368- na.name == " input" && storage_at (value_id) != *activation_storage;
1497+ (key == " et_vk.apply_rotary_emb_hf.default" ||
1498+ na.name == " input" ) &&
1499+ storage_at (value_id) != *activation_storage;
13691500 if (activation &&
13701501 (layout_at (value_id) != utils::kWidthPacked ||
13711502 storage_mismatch)) {
@@ -1639,6 +1770,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
16391770 detach_in_place_outputs ();
16401771 index_output_kinds ();
16411772 index_bindings ();
1773+ validate_marked_zero_constants (package);
16421774 plan_allocations ();
16431775
16441776 // 1. Materialize every native value as an ET-VK value.
0 commit comments