77#include < executorch/backends/native/runtime/vulkan/VulkanEngineExecutable.h>
88
99#include < algorithm>
10+ #include < any>
1011#include < array>
1112#include < cstring>
1213#include < deque>
3435#include < executorch/backends/native/runtime/graph/Value.h>
3536#include < executorch/backends/native/runtime/graph/utils/GraphUtils.h>
3637#include < executorch/backends/native/runtime/vulkan/VulkanConstantMaterializationTracker.h>
38+ #include < executorch/backends/native/runtime/vulkan/passes/FuseQuantizedLinear.h>
3739#include < executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
3840#include < executorch/backends/native/runtime/vulkan/passes/LowerRMSNorm.h>
3941#include < executorch/backends/native/runtime/vulkan/passes/MaterializeViewCopies.h>
@@ -450,9 +452,13 @@ class VulkanEngineExecutable final : public EngineExecutable {
450452 // pointer until prepack.
451453 std::deque<std::vector<uint8_t >> synth_bias_;
452454 std::vector<uint8_t > zero_state_;
453- // Canonical owner -> bytes shared by tensor references until ET-VK consumes
454- // and releases them during prepack.
455- std::unordered_map<std::string, std::shared_ptr<OwnedBytes>> constants_;
455+ // Bytes shared by tensor references until ET-VK consumes and releases them
456+ // during prepack: raw package bytes by canonical owner, and bytes computed
457+ // from them (int32-narrowed Long constants, q4 repacks) by derivation key.
458+ std::unordered_map<std::string, std::shared_ptr<OwnedBytes>>
459+ source_constants_;
460+ std::unordered_map<std::string, std::shared_ptr<OwnedBytes>>
461+ derived_constants_;
456462
457463 // vref_ and layout_ are sized to the method graph's value list, so every
458464 // ValueId the graph hands out addresses them. An id that does not means the
@@ -495,6 +501,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
495501 mutable_state_claim_(mutable_state_owners, method),
496502 graph_(std::make_unique<ComputeGraph>(config)) {
497503 method_.graph .rebuild_def_use ();
504+ vulkan::fuse_quantized_linears (method_);
498505 vulkan::lower_rms_norms (method_.graph );
499506 vulkan::insert_prepack_nodes (method_);
500507 validate_graph (method_.graph );
@@ -796,22 +803,18 @@ class VulkanEngineExecutable final : public EngineExecutable {
796803 std::to_string (info->nbytes ) + " bytes but its shape and dtype " +
797804 " need " + std::to_string (expected));
798805 }
799- std::shared_ptr<OwnedBytes>& storage = constants_[info->owner ];
800- if (!storage) {
801- std::optional<OwnedBytes> acquired =
802- package.acquire_constant (info->owner );
803- if (!acquired) {
804- throw std::runtime_error (
805- " vulkan: package could not load constant for key '" + b.key + " '" );
806- }
807- const ByteSpan source = acquired->span ();
808- if (source.size () != expected) {
809- throw std::runtime_error (
810- " vulkan: loaded constant '" + b.key + " ' has " +
811- std::to_string (source.size ()) + " bytes but metadata declares " +
812- std::to_string (expected));
813- }
814- if (info->dtype == ScalarType::Long) {
806+ std::shared_ptr<OwnedBytes> storage = source_constant (package, b);
807+ const ByteSpan source = storage->span ();
808+ if (source.size () != expected) {
809+ throw std::runtime_error (
810+ " vulkan: loaded constant '" + b.key + " ' has " +
811+ std::to_string (source.size ()) + " bytes but metadata declares " +
812+ std::to_string (expected));
813+ }
814+ if (info->dtype == ScalarType::Long) {
815+ std::shared_ptr<OwnedBytes>& narrowed = derived_constants_
816+ [std::to_string (info->owner .size ()) + " :" + info->owner + " :int32" ];
817+ if (!narrowed) {
815818 std::vector<uint8_t > converted (
816819 static_cast <size_t >(v.tensor_meta ().numel ()) * sizeof (int32_t ));
817820 for (size_t i = 0 ; i < converted.size () / sizeof (int32_t ); ++i) {
@@ -828,14 +831,220 @@ class VulkanEngineExecutable final : public EngineExecutable {
828831 &downcast,
829832 sizeof (downcast));
830833 }
831- storage = std::make_shared<OwnedBytes>(
834+ narrowed = std::make_shared<OwnedBytes>(
832835 OwnedBytes::from_vector (std::move (converted)));
833- } else {
834- storage = std::make_shared<OwnedBytes>(std::move (acquired.value ()));
835836 }
837+ storage = narrowed;
838+ }
839+ const ByteSpan bytes = storage->span ();
840+ auto * context = new std::shared_ptr<OwnedBytes>(storage);
841+ return executorch::runtime::FreeableBuffer (
842+ bytes.data (), bytes.size (), release_owned_bytes, context);
843+ }
844+
845+ // The only place package bytes are acquired, so each owner is recorded once.
846+ std::shared_ptr<OwnedBytes> source_constant (
847+ const Package& package,
848+ const DataBinding& binding) {
849+ const std::optional<ConstantInfo> info = package.constant_info (binding.key );
850+ if (!info) {
851+ throw std::runtime_error (
852+ " vulkan: package holds no source constant for key '" + binding.key +
853+ " '" );
854+ }
855+ std::shared_ptr<OwnedBytes>& storage = source_constants_[info->owner ];
856+ if (!storage) {
857+ std::optional<OwnedBytes> acquired =
858+ package.acquire_constant (info->owner );
859+ if (!acquired) {
860+ throw std::runtime_error (
861+ " vulkan: package could not load source constant for key '" +
862+ binding.key + " '" );
863+ }
864+ storage = std::make_shared<OwnedBytes>(std::move (acquired.value ()));
836865 materializations_.record (info->package_id , info->owner , info->nbytes );
837866 }
867+ return storage;
868+ }
869+
870+ void validate_zero_points (const Package& package, ValueId zero_points_id) {
871+ const DataBinding* binding = binding_at (zero_points_id);
872+ if (binding == nullptr || !binding->has_data ) {
873+ throw std::runtime_error (
874+ " vulkan: q4 weight zero points must be a stored constant" );
875+ }
876+ const std::shared_ptr<OwnedBytes> storage =
877+ source_constant (package, *binding);
878+ if (!std::ranges::all_of (
879+ storage->span (), [](uint8_t value) { return value == 0 ; })) {
880+ throw std::runtime_error (
881+ " vulkan: q4 weight zero points must all be zero" );
882+ }
883+ }
884+
885+ std::vector<uint8_t > pack_q4_weight (
886+ ByteSpan source,
887+ const ConstantInfo& info,
888+ const TensorMeta& output) {
889+ if (info.dtype != ScalarType::Char || info.sizes == nullptr ||
890+ info.sizes ->size () != 2 || output.dtype != ScalarType::Byte ||
891+ output.sizes .size () != 2 || info.sizes ->at (0 ) != output.sizes [0 ] ||
892+ info.sizes ->at (1 ) != output.sizes [1 ] * 2 ||
893+ source.size () !=
894+ static_cast <size_t >(info.sizes ->at (0 ) * info.sizes ->at (1 ))) {
895+ throw std::runtime_error (
896+ " vulkan: invalid source metadata for q4 packing" );
897+ }
898+ std::vector<uint8_t > packed (logical_nbytes (output));
899+ for (size_t i = 0 ; i < packed.size (); ++i) {
900+ const int8_t even = static_cast <int8_t >(source[i * 2 ]);
901+ const int8_t odd = static_cast <int8_t >(source[i * 2 + 1 ]);
902+ if (even < -8 || even > 7 || odd < -8 || odd > 7 ) {
903+ throw std::runtime_error (" vulkan: q4 weight value is outside [-8, 7]" );
904+ }
905+ packed[i] = static_cast <uint8_t >(
906+ (static_cast <uint8_t >(odd + 8 ) << 4 ) |
907+ static_cast <uint8_t >(even + 8 ));
908+ }
909+ return packed;
910+ }
911+
912+ std::vector<uint8_t > transpose_q4_scales (
913+ ByteSpan source,
914+ const ConstantInfo& info,
915+ const TensorMeta& output) {
916+ if ((info.dtype != ScalarType::Float && info.dtype != ScalarType::Half) ||
917+ info.sizes == nullptr || info.sizes ->size () != 2 ||
918+ output.dtype != info.dtype || output.sizes .size () != 2 ||
919+ info.sizes ->at (0 ) > output.sizes [1 ] ||
920+ info.sizes ->at (1 ) != output.sizes [0 ]) {
921+ throw std::runtime_error (
922+ " vulkan: invalid source metadata for q4 scale transpose" );
923+ }
924+ const size_t elem_size = element_size (info.dtype );
925+ const size_t rows = static_cast <size_t >(info.sizes ->at (0 ));
926+ const size_t cols = static_cast <size_t >(info.sizes ->at (1 ));
927+ const size_t output_cols = static_cast <size_t >(output.sizes [1 ]);
928+ if (source.size () != rows * cols * elem_size) {
929+ throw std::runtime_error (" vulkan: q4 scale source byte count is invalid" );
930+ }
931+ std::vector<uint8_t > transposed (logical_nbytes (output), 0 );
932+ for (size_t row = 0 ; row < rows; ++row) {
933+ for (size_t col = 0 ; col < cols; ++col) {
934+ std::memcpy (
935+ transposed.data () + (col * output_cols + row) * elem_size,
936+ source.data () + (row * cols + col) * elem_size,
937+ elem_size);
938+ }
939+ }
940+ return transposed;
941+ }
942+
943+ std::vector<uint8_t > q4_weight_sums (
944+ ByteSpan source,
945+ const ConstantInfo& info,
946+ const TensorMeta& output,
947+ int64_t group_size) {
948+ if (info.sizes == nullptr || info.sizes ->size () != 2 ||
949+ output.dtype != ScalarType::Int || output.sizes .size () != 2 ||
950+ group_size <= 0 || info.sizes ->at (0 ) > output.sizes [1 ]) {
951+ throw std::runtime_error (
952+ " vulkan: invalid source metadata for q4 group sums" );
953+ }
954+ const size_t rows = static_cast <size_t >(info.sizes ->at (0 ));
955+ const size_t groups = static_cast <size_t >(output.sizes [0 ]);
956+ const size_t cols = groups * static_cast <size_t >(group_size);
957+ const size_t output_cols = static_cast <size_t >(output.sizes [1 ]);
958+ const bool packed = info.dtype == ScalarType::Byte;
959+ const size_t stored_cols = static_cast <size_t >(info.sizes ->at (1 ));
960+ if ((!packed && info.dtype != ScalarType::Char) ||
961+ stored_cols * (packed ? 2 : 1 ) != cols ||
962+ source.size () != rows * stored_cols) {
963+ throw std::runtime_error (
964+ " vulkan: invalid source metadata for q4 group sums" );
965+ }
966+ std::vector<uint8_t > sums (logical_nbytes (output), 0 );
967+ for (size_t row = 0 ; row < rows; ++row) {
968+ for (size_t group = 0 ; group < groups; ++group) {
969+ int32_t sum = 0 ;
970+ const size_t start = group * static_cast <size_t >(group_size);
971+ for (size_t i = 0 ; i < static_cast <size_t >(group_size); ++i) {
972+ const size_t col = start + i;
973+ if (packed) {
974+ const uint8_t byte = source[row * stored_cols + col / 2 ];
975+ const uint8_t nibble = col % 2 == 0 ? byte & 0x0F : byte >> 4 ;
976+ sum += static_cast <int32_t >(nibble) - 8 ;
977+ } else {
978+ sum += static_cast <int8_t >(source[row * stored_cols + col]);
979+ }
980+ }
981+ std::memcpy (
982+ sums.data () + (group * output_cols + row) * sizeof (sum),
983+ &sum,
984+ sizeof (sum));
985+ }
986+ }
987+ return sums;
988+ }
989+
990+ executorch::runtime::FreeableBuffer derived_q4_constant_buffer (
991+ const Package& package,
992+ const Value& value,
993+ const vulkan::Q4ConstantTransform& transform) {
994+ const DataBinding* source_binding = binding_at (transform.source_id );
995+ if (source_binding == nullptr || !source_binding->has_data ) {
996+ throw std::runtime_error (
997+ " vulkan: derived q4 value has no stored source constant" );
998+ }
999+ const std::optional<ConstantInfo> info =
1000+ package.constant_info (source_binding->key );
1001+ if (!info) {
1002+ throw std::runtime_error (
1003+ " vulkan: package holds no q4 source constant for key '" +
1004+ source_binding->key + " '" );
1005+ }
1006+ const std::string cache_key = std::to_string (info->owner .size ()) + " :" +
1007+ info->owner +
1008+ " :q4:" + std::to_string (static_cast <int >(transform.kind )) + " :" +
1009+ value.name ;
1010+ std::shared_ptr<OwnedBytes>& storage = derived_constants_[cache_key];
1011+ if (!storage) {
1012+ const std::shared_ptr<OwnedBytes> source =
1013+ source_constant (package, *source_binding);
1014+ std::vector<uint8_t > transformed;
1015+ switch (transform.kind ) {
1016+ case vulkan::Q4ConstantTransformKind::PackWeight:
1017+ validate_zero_points (package, transform.zero_points_id );
1018+ if (info->dtype == ScalarType::Byte && info->sizes != nullptr &&
1019+ *info->sizes == value.tensor_meta ().sizes &&
1020+ info->nbytes == logical_nbytes (value.tensor_meta ())) {
1021+ storage = source;
1022+ } else {
1023+ transformed =
1024+ pack_q4_weight (source->span (), *info, value.tensor_meta ());
1025+ }
1026+ break ;
1027+ case vulkan::Q4ConstantTransformKind::TransposeScales:
1028+ transformed =
1029+ transpose_q4_scales (source->span (), *info, value.tensor_meta ());
1030+ break ;
1031+ case vulkan::Q4ConstantTransformKind::WeightSums:
1032+ validate_zero_points (package, transform.zero_points_id );
1033+ transformed = q4_weight_sums (
1034+ source->span (), *info, value.tensor_meta (), transform.group_size );
1035+ break ;
1036+ }
1037+ if (!storage) {
1038+ storage = std::make_shared<OwnedBytes>(
1039+ OwnedBytes::from_vector (std::move (transformed)));
1040+ }
1041+ }
8381042 const ByteSpan bytes = storage->span ();
1043+ if (bytes.size () != logical_nbytes (value.tensor_meta ())) {
1044+ throw std::runtime_error (
1045+ " vulkan: derived q4 constant '" + value.name +
1046+ " ' does not match its declared size" );
1047+ }
8391048 auto * context = new std::shared_ptr<OwnedBytes>(storage);
8401049 return executorch::runtime::FreeableBuffer (
8411050 bytes.data (), bytes.size (), release_owned_bytes, context);
@@ -868,6 +1077,19 @@ class VulkanEngineExecutable final : public EngineExecutable {
8681077 case ValueKind::Tensor: {
8691078 const TensorMeta& m = v.tensor_meta ();
8701079 const std::vector<int64_t >& sizes = m.sizes ;
1080+ const auto transform = v.attrs .find (vulkan::kQ4ConstantTransformAttr );
1081+ if (transform != v.attrs .end ()) {
1082+ const auto * q4 =
1083+ std::any_cast<vulkan::Q4ConstantTransform>(&transform->second );
1084+ if (q4 == nullptr ) {
1085+ throw std::runtime_error (
1086+ " vulkan: malformed q4 constant transform annotation" );
1087+ }
1088+ return graph_->add_tensorref (
1089+ sizes,
1090+ to_vk_dtype (device_dtype (m.dtype )),
1091+ derived_q4_constant_buffer (package, v, *q4));
1092+ }
8711093 if (valid (v.alias_id )) {
8721094 const VkRef source = vref_at (v.alias_id );
8731095 const TensorMeta& source_meta = g ().value (v.alias_id ).tensor_meta ();
@@ -1440,7 +1662,8 @@ class VulkanEngineExecutable final : public EngineExecutable {
14401662
14411663 // 5. Finalize and upload constants. TensorRef owns the remaining shared
14421664 // references and releases each source after its final prepack consumer.
1443- constants_.clear ();
1665+ source_constants_.clear ();
1666+ derived_constants_.clear ();
14441667 graph_->prepare ();
14451668 graph_->prepare_pipelines ();
14461669 graph_->prepack ();
0 commit comments