Skip to content

Commit cbdaae1

Browse files
[executorch][native-vk] Lower portable q4 linears at load time (pytorch#23467)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23432 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/127/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/127/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/126/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/127/orig Differential Revision: [D121156276](https://our.internmc.facebook.com/intern/diff/D121156276/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent ba4bab8 commit cbdaae1

6 files changed

Lines changed: 900 additions & 23 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 246 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <executorch/backends/native/runtime/vulkan/VulkanEngineExecutable.h>
88

99
#include <algorithm>
10+
#include <any>
1011
#include <array>
1112
#include <cstring>
1213
#include <deque>
@@ -34,6 +35,7 @@
3435
#include <executorch/backends/native/runtime/graph/Value.h>
3536
#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h>
3637
#include <executorch/backends/native/runtime/vulkan/VulkanConstantMaterializationTracker.h>
38+
#include <executorch/backends/native/runtime/vulkan/passes/FuseQuantizedLinear.h>
3739
#include <executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
3840
#include <executorch/backends/native/runtime/vulkan/passes/LowerRMSNorm.h>
3941
#include <executorch/backends/native/runtime/vulkan/passes/MaterializeViewCopies.h>
@@ -450,9 +452,13 @@ class VulkanEngineExecutable final : public EngineExecutable {
450452
// pointer until prepack.
451453
std::deque<std::vector<uint8_t>> synth_bias_;
452454
std::vector<uint8_t> zero_state_;
453-
// Canonical owner -> bytes shared by tensor references until ET-VK consumes
454-
// and releases them during prepack.
455-
std::unordered_map<std::string, std::shared_ptr<OwnedBytes>> constants_;
455+
// Bytes shared by tensor references until ET-VK consumes and releases them
456+
// during prepack: raw package bytes by canonical owner, and bytes computed
457+
// from them (int32-narrowed Long constants, q4 repacks) by derivation key.
458+
std::unordered_map<std::string, std::shared_ptr<OwnedBytes>>
459+
source_constants_;
460+
std::unordered_map<std::string, std::shared_ptr<OwnedBytes>>
461+
derived_constants_;
456462

457463
// vref_ and layout_ are sized to the method graph's value list, so every
458464
// ValueId the graph hands out addresses them. An id that does not means the
@@ -495,6 +501,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
495501
mutable_state_claim_(mutable_state_owners, method),
496502
graph_(std::make_unique<ComputeGraph>(config)) {
497503
method_.graph.rebuild_def_use();
504+
vulkan::fuse_quantized_linears(method_);
498505
vulkan::lower_rms_norms(method_.graph);
499506
vulkan::insert_prepack_nodes(method_);
500507
validate_graph(method_.graph);
@@ -796,22 +803,18 @@ class VulkanEngineExecutable final : public EngineExecutable {
796803
std::to_string(info->nbytes) + " bytes but its shape and dtype " +
797804
"need " + std::to_string(expected));
798805
}
799-
std::shared_ptr<OwnedBytes>& storage = constants_[info->owner];
800-
if (!storage) {
801-
std::optional<OwnedBytes> acquired =
802-
package.acquire_constant(info->owner);
803-
if (!acquired) {
804-
throw std::runtime_error(
805-
"vulkan: package could not load constant for key '" + b.key + "'");
806-
}
807-
const ByteSpan source = acquired->span();
808-
if (source.size() != expected) {
809-
throw std::runtime_error(
810-
"vulkan: loaded constant '" + b.key + "' has " +
811-
std::to_string(source.size()) + " bytes but metadata declares " +
812-
std::to_string(expected));
813-
}
814-
if (info->dtype == ScalarType::Long) {
806+
std::shared_ptr<OwnedBytes> storage = source_constant(package, b);
807+
const ByteSpan source = storage->span();
808+
if (source.size() != expected) {
809+
throw std::runtime_error(
810+
"vulkan: loaded constant '" + b.key + "' has " +
811+
std::to_string(source.size()) + " bytes but metadata declares " +
812+
std::to_string(expected));
813+
}
814+
if (info->dtype == ScalarType::Long) {
815+
std::shared_ptr<OwnedBytes>& narrowed = derived_constants_
816+
[std::to_string(info->owner.size()) + ":" + info->owner + ":int32"];
817+
if (!narrowed) {
815818
std::vector<uint8_t> converted(
816819
static_cast<size_t>(v.tensor_meta().numel()) * sizeof(int32_t));
817820
for (size_t i = 0; i < converted.size() / sizeof(int32_t); ++i) {
@@ -828,14 +831,220 @@ class VulkanEngineExecutable final : public EngineExecutable {
828831
&downcast,
829832
sizeof(downcast));
830833
}
831-
storage = std::make_shared<OwnedBytes>(
834+
narrowed = std::make_shared<OwnedBytes>(
832835
OwnedBytes::from_vector(std::move(converted)));
833-
} else {
834-
storage = std::make_shared<OwnedBytes>(std::move(acquired.value()));
835836
}
837+
storage = narrowed;
838+
}
839+
const ByteSpan bytes = storage->span();
840+
auto* context = new std::shared_ptr<OwnedBytes>(storage);
841+
return executorch::runtime::FreeableBuffer(
842+
bytes.data(), bytes.size(), release_owned_bytes, context);
843+
}
844+
845+
// The only place package bytes are acquired, so each owner is recorded once.
846+
std::shared_ptr<OwnedBytes> source_constant(
847+
const Package& package,
848+
const DataBinding& binding) {
849+
const std::optional<ConstantInfo> info = package.constant_info(binding.key);
850+
if (!info) {
851+
throw std::runtime_error(
852+
"vulkan: package holds no source constant for key '" + binding.key +
853+
"'");
854+
}
855+
std::shared_ptr<OwnedBytes>& storage = source_constants_[info->owner];
856+
if (!storage) {
857+
std::optional<OwnedBytes> acquired =
858+
package.acquire_constant(info->owner);
859+
if (!acquired) {
860+
throw std::runtime_error(
861+
"vulkan: package could not load source constant for key '" +
862+
binding.key + "'");
863+
}
864+
storage = std::make_shared<OwnedBytes>(std::move(acquired.value()));
836865
materializations_.record(info->package_id, info->owner, info->nbytes);
837866
}
867+
return storage;
868+
}
869+
870+
void validate_zero_points(const Package& package, ValueId zero_points_id) {
871+
const DataBinding* binding = binding_at(zero_points_id);
872+
if (binding == nullptr || !binding->has_data) {
873+
throw std::runtime_error(
874+
"vulkan: q4 weight zero points must be a stored constant");
875+
}
876+
const std::shared_ptr<OwnedBytes> storage =
877+
source_constant(package, *binding);
878+
if (!std::ranges::all_of(
879+
storage->span(), [](uint8_t value) { return value == 0; })) {
880+
throw std::runtime_error(
881+
"vulkan: q4 weight zero points must all be zero");
882+
}
883+
}
884+
885+
std::vector<uint8_t> pack_q4_weight(
886+
ByteSpan source,
887+
const ConstantInfo& info,
888+
const TensorMeta& output) {
889+
if (info.dtype != ScalarType::Char || info.sizes == nullptr ||
890+
info.sizes->size() != 2 || output.dtype != ScalarType::Byte ||
891+
output.sizes.size() != 2 || info.sizes->at(0) != output.sizes[0] ||
892+
info.sizes->at(1) != output.sizes[1] * 2 ||
893+
source.size() !=
894+
static_cast<size_t>(info.sizes->at(0) * info.sizes->at(1))) {
895+
throw std::runtime_error(
896+
"vulkan: invalid source metadata for q4 packing");
897+
}
898+
std::vector<uint8_t> packed(logical_nbytes(output));
899+
for (size_t i = 0; i < packed.size(); ++i) {
900+
const int8_t even = static_cast<int8_t>(source[i * 2]);
901+
const int8_t odd = static_cast<int8_t>(source[i * 2 + 1]);
902+
if (even < -8 || even > 7 || odd < -8 || odd > 7) {
903+
throw std::runtime_error("vulkan: q4 weight value is outside [-8, 7]");
904+
}
905+
packed[i] = static_cast<uint8_t>(
906+
(static_cast<uint8_t>(odd + 8) << 4) |
907+
static_cast<uint8_t>(even + 8));
908+
}
909+
return packed;
910+
}
911+
912+
std::vector<uint8_t> transpose_q4_scales(
913+
ByteSpan source,
914+
const ConstantInfo& info,
915+
const TensorMeta& output) {
916+
if ((info.dtype != ScalarType::Float && info.dtype != ScalarType::Half) ||
917+
info.sizes == nullptr || info.sizes->size() != 2 ||
918+
output.dtype != info.dtype || output.sizes.size() != 2 ||
919+
info.sizes->at(0) > output.sizes[1] ||
920+
info.sizes->at(1) != output.sizes[0]) {
921+
throw std::runtime_error(
922+
"vulkan: invalid source metadata for q4 scale transpose");
923+
}
924+
const size_t elem_size = element_size(info.dtype);
925+
const size_t rows = static_cast<size_t>(info.sizes->at(0));
926+
const size_t cols = static_cast<size_t>(info.sizes->at(1));
927+
const size_t output_cols = static_cast<size_t>(output.sizes[1]);
928+
if (source.size() != rows * cols * elem_size) {
929+
throw std::runtime_error("vulkan: q4 scale source byte count is invalid");
930+
}
931+
std::vector<uint8_t> transposed(logical_nbytes(output), 0);
932+
for (size_t row = 0; row < rows; ++row) {
933+
for (size_t col = 0; col < cols; ++col) {
934+
std::memcpy(
935+
transposed.data() + (col * output_cols + row) * elem_size,
936+
source.data() + (row * cols + col) * elem_size,
937+
elem_size);
938+
}
939+
}
940+
return transposed;
941+
}
942+
943+
std::vector<uint8_t> q4_weight_sums(
944+
ByteSpan source,
945+
const ConstantInfo& info,
946+
const TensorMeta& output,
947+
int64_t group_size) {
948+
if (info.sizes == nullptr || info.sizes->size() != 2 ||
949+
output.dtype != ScalarType::Int || output.sizes.size() != 2 ||
950+
group_size <= 0 || info.sizes->at(0) > output.sizes[1]) {
951+
throw std::runtime_error(
952+
"vulkan: invalid source metadata for q4 group sums");
953+
}
954+
const size_t rows = static_cast<size_t>(info.sizes->at(0));
955+
const size_t groups = static_cast<size_t>(output.sizes[0]);
956+
const size_t cols = groups * static_cast<size_t>(group_size);
957+
const size_t output_cols = static_cast<size_t>(output.sizes[1]);
958+
const bool packed = info.dtype == ScalarType::Byte;
959+
const size_t stored_cols = static_cast<size_t>(info.sizes->at(1));
960+
if ((!packed && info.dtype != ScalarType::Char) ||
961+
stored_cols * (packed ? 2 : 1) != cols ||
962+
source.size() != rows * stored_cols) {
963+
throw std::runtime_error(
964+
"vulkan: invalid source metadata for q4 group sums");
965+
}
966+
std::vector<uint8_t> sums(logical_nbytes(output), 0);
967+
for (size_t row = 0; row < rows; ++row) {
968+
for (size_t group = 0; group < groups; ++group) {
969+
int32_t sum = 0;
970+
const size_t start = group * static_cast<size_t>(group_size);
971+
for (size_t i = 0; i < static_cast<size_t>(group_size); ++i) {
972+
const size_t col = start + i;
973+
if (packed) {
974+
const uint8_t byte = source[row * stored_cols + col / 2];
975+
const uint8_t nibble = col % 2 == 0 ? byte & 0x0F : byte >> 4;
976+
sum += static_cast<int32_t>(nibble) - 8;
977+
} else {
978+
sum += static_cast<int8_t>(source[row * stored_cols + col]);
979+
}
980+
}
981+
std::memcpy(
982+
sums.data() + (group * output_cols + row) * sizeof(sum),
983+
&sum,
984+
sizeof(sum));
985+
}
986+
}
987+
return sums;
988+
}
989+
990+
executorch::runtime::FreeableBuffer derived_q4_constant_buffer(
991+
const Package& package,
992+
const Value& value,
993+
const vulkan::Q4ConstantTransform& transform) {
994+
const DataBinding* source_binding = binding_at(transform.source_id);
995+
if (source_binding == nullptr || !source_binding->has_data) {
996+
throw std::runtime_error(
997+
"vulkan: derived q4 value has no stored source constant");
998+
}
999+
const std::optional<ConstantInfo> info =
1000+
package.constant_info(source_binding->key);
1001+
if (!info) {
1002+
throw std::runtime_error(
1003+
"vulkan: package holds no q4 source constant for key '" +
1004+
source_binding->key + "'");
1005+
}
1006+
const std::string cache_key = std::to_string(info->owner.size()) + ":" +
1007+
info->owner +
1008+
":q4:" + std::to_string(static_cast<int>(transform.kind)) + ":" +
1009+
value.name;
1010+
std::shared_ptr<OwnedBytes>& storage = derived_constants_[cache_key];
1011+
if (!storage) {
1012+
const std::shared_ptr<OwnedBytes> source =
1013+
source_constant(package, *source_binding);
1014+
std::vector<uint8_t> transformed;
1015+
switch (transform.kind) {
1016+
case vulkan::Q4ConstantTransformKind::PackWeight:
1017+
validate_zero_points(package, transform.zero_points_id);
1018+
if (info->dtype == ScalarType::Byte && info->sizes != nullptr &&
1019+
*info->sizes == value.tensor_meta().sizes &&
1020+
info->nbytes == logical_nbytes(value.tensor_meta())) {
1021+
storage = source;
1022+
} else {
1023+
transformed =
1024+
pack_q4_weight(source->span(), *info, value.tensor_meta());
1025+
}
1026+
break;
1027+
case vulkan::Q4ConstantTransformKind::TransposeScales:
1028+
transformed =
1029+
transpose_q4_scales(source->span(), *info, value.tensor_meta());
1030+
break;
1031+
case vulkan::Q4ConstantTransformKind::WeightSums:
1032+
validate_zero_points(package, transform.zero_points_id);
1033+
transformed = q4_weight_sums(
1034+
source->span(), *info, value.tensor_meta(), transform.group_size);
1035+
break;
1036+
}
1037+
if (!storage) {
1038+
storage = std::make_shared<OwnedBytes>(
1039+
OwnedBytes::from_vector(std::move(transformed)));
1040+
}
1041+
}
8381042
const ByteSpan bytes = storage->span();
1043+
if (bytes.size() != logical_nbytes(value.tensor_meta())) {
1044+
throw std::runtime_error(
1045+
"vulkan: derived q4 constant '" + value.name +
1046+
"' does not match its declared size");
1047+
}
8391048
auto* context = new std::shared_ptr<OwnedBytes>(storage);
8401049
return executorch::runtime::FreeableBuffer(
8411050
bytes.data(), bytes.size(), release_owned_bytes, context);
@@ -868,6 +1077,19 @@ class VulkanEngineExecutable final : public EngineExecutable {
8681077
case ValueKind::Tensor: {
8691078
const TensorMeta& m = v.tensor_meta();
8701079
const std::vector<int64_t>& sizes = m.sizes;
1080+
const auto transform = v.attrs.find(vulkan::kQ4ConstantTransformAttr);
1081+
if (transform != v.attrs.end()) {
1082+
const auto* q4 =
1083+
std::any_cast<vulkan::Q4ConstantTransform>(&transform->second);
1084+
if (q4 == nullptr) {
1085+
throw std::runtime_error(
1086+
"vulkan: malformed q4 constant transform annotation");
1087+
}
1088+
return graph_->add_tensorref(
1089+
sizes,
1090+
to_vk_dtype(device_dtype(m.dtype)),
1091+
derived_q4_constant_buffer(package, v, *q4));
1092+
}
8711093
if (valid(v.alias_id)) {
8721094
const VkRef source = vref_at(v.alias_id);
8731095
const TensorMeta& source_meta = g().value(v.alias_id).tensor_meta();
@@ -1440,7 +1662,8 @@ class VulkanEngineExecutable final : public EngineExecutable {
14401662

14411663
// 5. Finalize and upload constants. TensorRef owns the remaining shared
14421664
// references and releases each source after its final prepack consumer.
1443-
constants_.clear();
1665+
source_constants_.clear();
1666+
derived_constants_.clear();
14441667
graph_->prepare();
14451668
graph_->prepare_pipelines();
14461669
graph_->prepack();

0 commit comments

Comments
 (0)