Skip to content

Commit ab46560

Browse files
[executorch][native-vk] Sum q4 weight groups a byte at a time (pytorch#23471)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23437 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/131/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/131/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/130/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/131/orig Differential Revision: [D121704328](https://our.internmc.facebook.com/intern/diff/D121704328/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent 1ec95bf commit ab46560

4 files changed

Lines changed: 122 additions & 22 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1039,28 +1039,16 @@ class VulkanEngineExecutable final : public EngineExecutable {
10391039
throw std::runtime_error(
10401040
"vulkan: invalid source metadata for q4 group sums");
10411041
}
1042-
std::vector<uint8_t> sums(logical_nbytes(output), 0);
1043-
for (size_t row = 0; row < rows; ++row) {
1044-
for (size_t group = 0; group < groups; ++group) {
1045-
int32_t sum = 0;
1046-
const size_t start = group * static_cast<size_t>(group_size);
1047-
for (size_t i = 0; i < static_cast<size_t>(group_size); ++i) {
1048-
const size_t col = start + i;
1049-
if (packed) {
1050-
const uint8_t byte = source[row * stored_cols + col / 2];
1051-
const uint8_t nibble = col % 2 == 0 ? byte & 0x0F : byte >> 4;
1052-
sum += static_cast<int32_t>(nibble) - 8;
1053-
} else {
1054-
sum += static_cast<int8_t>(source[row * stored_cols + col]);
1055-
}
1056-
}
1057-
std::memcpy(
1058-
sums.data() + (group * output_cols + row) * sizeof(sum),
1059-
&sum,
1060-
sizeof(sum));
1061-
}
1062-
}
1063-
return sums;
1042+
const std::vector<int32_t> sums = vulkan::q4_group_sums(
1043+
source,
1044+
{.rows = rows,
1045+
.cols = cols,
1046+
.group_size = static_cast<size_t>(group_size),
1047+
.output_cols = output_cols,
1048+
.packed = packed});
1049+
std::vector<uint8_t> bytes(logical_nbytes(output));
1050+
std::memcpy(bytes.data(), sums.data(), bytes.size());
1051+
return bytes;
10641052
}
10651053

10661054
executorch::runtime::FreeableBuffer derived_q4_constant_buffer(

‎backends/native/runtime/vulkan/passes/FuseQuantizedLinear.cpp‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,34 @@
2020
namespace ptn::vulkan {
2121
namespace {
2222

23+
int32_t packed_q4_group_sum(const uint8_t* row, size_t start, size_t count) {
24+
const size_t end = start + count;
25+
size_t col = start;
26+
int32_t sum = 0;
27+
if (col % 2 != 0) {
28+
sum += row[col / 2] >> 4;
29+
++col;
30+
}
31+
const uint8_t* bytes = row + col / 2;
32+
const size_t whole_bytes = (end - col) / 2;
33+
for (size_t i = 0; i < whole_bytes; ++i) {
34+
sum += (bytes[i] & 0x0F) + (bytes[i] >> 4);
35+
}
36+
col += 2 * whole_bytes;
37+
if (col < end) {
38+
sum += row[col / 2] & 0x0F;
39+
}
40+
return sum - 8 * static_cast<int32_t>(count);
41+
}
42+
43+
int32_t int8_group_sum(const uint8_t* row, size_t start, size_t count) {
44+
int32_t sum = 0;
45+
for (size_t col = start; col < start + count; ++col) {
46+
sum += static_cast<int8_t>(row[col]);
47+
}
48+
return sum;
49+
}
50+
2351
constexpr std::string_view kChooseQParams =
2452
"torch.ops.torchao.choose_qparams_affine.default";
2553
constexpr std::string_view kQuantize =
@@ -359,6 +387,24 @@ bool rewrite_match(
359387

360388
} // namespace
361389

390+
std::vector<int32_t> q4_group_sums(
391+
std::span<const uint8_t> weight,
392+
const Q4GroupSumsLayout& layout) {
393+
const size_t groups = layout.cols / layout.group_size;
394+
const size_t stored_cols = layout.packed ? layout.cols / 2 : layout.cols;
395+
std::vector<int32_t> sums(groups * layout.output_cols, 0);
396+
for (size_t row = 0; row < layout.rows; ++row) {
397+
const uint8_t* row_bytes = weight.data() + row * stored_cols;
398+
for (size_t group = 0; group < groups; ++group) {
399+
const size_t start = group * layout.group_size;
400+
sums[group * layout.output_cols + row] = layout.packed
401+
? packed_q4_group_sum(row_bytes, start, layout.group_size)
402+
: int8_group_sum(row_bytes, start, layout.group_size);
403+
}
404+
}
405+
return sums;
406+
}
407+
362408
size_t fuse_quantized_linears(Method& method) {
363409
Graph& graph = method.graph;
364410
graph.rebuild_def_use();

‎backends/native/runtime/vulkan/passes/FuseQuantizedLinear.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
#pragma once
88

99
#include <cstddef>
10+
#include <cstdint>
11+
#include <span>
12+
#include <vector>
1013

1114
#include <executorch/backends/native/runtime/Method.h>
1215

@@ -28,6 +31,22 @@ struct Q4ConstantTransform {
2831
int64_t group_size = 0;
2932
};
3033

34+
struct Q4GroupSumsLayout {
35+
size_t rows;
36+
size_t cols;
37+
size_t group_size;
38+
size_t output_cols;
39+
// Two weights per byte, low nibble first, each stored as value + 8.
40+
// Otherwise one int8 weight per byte.
41+
bool packed;
42+
};
43+
44+
// Returns the int32 sum of each [row, group] of `weight` as
45+
// [cols / group_size, output_cols]. Columns past `rows` are zero.
46+
std::vector<int32_t> q4_group_sums(
47+
std::span<const uint8_t> weight,
48+
const Q4GroupSumsLayout& layout);
49+
3150
// Rewrites portable torchao q8-dynamic/q4-weight linear patterns into the
3251
// Native-VK runtime kernel. This runs after PTN deserialization; PTNs therefore
3352
// contain neither et_vk operators nor Vulkan AOT transformations.

‎backends/native/test/runtime/vulkan/test_fuse_quantized_linear.cpp‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
#include <algorithm>
1212
#include <any>
13+
#include <cstdint>
1314
#include <string>
1415
#include <vector>
1516

@@ -163,6 +164,14 @@ const Q4ConstantTransform& transform_of(const Value& value) {
163164
value.attrs.at(kQ4ConstantTransformAttr));
164165
}
165166

167+
std::vector<uint8_t> pack_q4(const std::vector<int8_t>& weight) {
168+
std::vector<uint8_t> packed;
169+
for (size_t i = 0; i < weight.size(); i += 2) {
170+
packed.push_back((weight[i] + 8) | ((weight[i + 1] + 8) << 4));
171+
}
172+
return packed;
173+
}
174+
166175
TEST(FuseQuantizedLinearTest, RewritesPortablePatternAtRuntime) {
167176
Method method = make_dynamic_q4_linear();
168177
EXPECT_EQ(fuse_quantized_linears(method), 1);
@@ -225,5 +234,43 @@ TEST(FuseQuantizedLinearTest, PreservesHalfPrecisionScales) {
225234
EXPECT_EQ(method.graph.value(2).tensor_meta().dtype, ScalarType::Half);
226235
}
227236

237+
TEST(FuseQuantizedLinearTest, SumsQ4WeightGroups) {
238+
// Two rows of six q4 weights, laid out as [rows, cols].
239+
const std::vector<int8_t> weight = {-8, 7, 1, 2, -3, 0, 3, 3, 3, -1, -1, -1};
240+
const std::vector<uint8_t> packed = pack_q4(weight);
241+
const std::vector<uint8_t> unpacked(weight.begin(), weight.end());
242+
// [groups, output_cols], with a zero column past the last row.
243+
const std::vector<int32_t> odd_group_sums = {0, 9, 0, -1, -3, 0};
244+
const std::vector<int32_t> even_group_sums = {-1, 6, 3, 2, -3, -2};
245+
246+
EXPECT_EQ(
247+
q4_group_sums(
248+
packed,
249+
{.rows = 2,
250+
.cols = 6,
251+
.group_size = 3,
252+
.output_cols = 3,
253+
.packed = true}),
254+
odd_group_sums);
255+
EXPECT_EQ(
256+
q4_group_sums(
257+
unpacked,
258+
{.rows = 2,
259+
.cols = 6,
260+
.group_size = 3,
261+
.output_cols = 3,
262+
.packed = false}),
263+
odd_group_sums);
264+
EXPECT_EQ(
265+
q4_group_sums(
266+
packed,
267+
{.rows = 2,
268+
.cols = 6,
269+
.group_size = 2,
270+
.output_cols = 2,
271+
.packed = true}),
272+
even_group_sums);
273+
}
274+
228275
} // namespace
229276
} // namespace ptn::vulkan

0 commit comments

Comments
 (0)