|
20 | 20 | namespace ptn::vulkan { |
21 | 21 | namespace { |
22 | 22 |
|
| 23 | +int32_t packed_q4_group_sum(const uint8_t* row, size_t start, size_t count) { |
| 24 | + const size_t end = start + count; |
| 25 | + size_t col = start; |
| 26 | + int32_t sum = 0; |
| 27 | + if (col % 2 != 0) { |
| 28 | + sum += row[col / 2] >> 4; |
| 29 | + ++col; |
| 30 | + } |
| 31 | + const uint8_t* bytes = row + col / 2; |
| 32 | + const size_t whole_bytes = (end - col) / 2; |
| 33 | + for (size_t i = 0; i < whole_bytes; ++i) { |
| 34 | + sum += (bytes[i] & 0x0F) + (bytes[i] >> 4); |
| 35 | + } |
| 36 | + col += 2 * whole_bytes; |
| 37 | + if (col < end) { |
| 38 | + sum += row[col / 2] & 0x0F; |
| 39 | + } |
| 40 | + return sum - 8 * static_cast<int32_t>(count); |
| 41 | +} |
| 42 | + |
| 43 | +int32_t int8_group_sum(const uint8_t* row, size_t start, size_t count) { |
| 44 | + int32_t sum = 0; |
| 45 | + for (size_t col = start; col < start + count; ++col) { |
| 46 | + sum += static_cast<int8_t>(row[col]); |
| 47 | + } |
| 48 | + return sum; |
| 49 | +} |
| 50 | + |
23 | 51 | constexpr std::string_view kChooseQParams = |
24 | 52 | "torch.ops.torchao.choose_qparams_affine.default"; |
25 | 53 | constexpr std::string_view kQuantize = |
@@ -359,6 +387,24 @@ bool rewrite_match( |
359 | 387 |
|
360 | 388 | } // namespace |
361 | 389 |
|
| 390 | +std::vector<int32_t> q4_group_sums( |
| 391 | + std::span<const uint8_t> weight, |
| 392 | + const Q4GroupSumsLayout& layout) { |
| 393 | + const size_t groups = layout.cols / layout.group_size; |
| 394 | + const size_t stored_cols = layout.packed ? layout.cols / 2 : layout.cols; |
| 395 | + std::vector<int32_t> sums(groups * layout.output_cols, 0); |
| 396 | + for (size_t row = 0; row < layout.rows; ++row) { |
| 397 | + const uint8_t* row_bytes = weight.data() + row * stored_cols; |
| 398 | + for (size_t group = 0; group < groups; ++group) { |
| 399 | + const size_t start = group * layout.group_size; |
| 400 | + sums[group * layout.output_cols + row] = layout.packed |
| 401 | + ? packed_q4_group_sum(row_bytes, start, layout.group_size) |
| 402 | + : int8_group_sum(row_bytes, start, layout.group_size); |
| 403 | + } |
| 404 | + } |
| 405 | + return sums; |
| 406 | +} |
| 407 | + |
362 | 408 | size_t fuse_quantized_linears(Method& method) { |
363 | 409 | Graph& graph = method.graph; |
364 | 410 | graph.rebuild_def_use(); |
|
0 commit comments