Skip to content

Commit f1adb8b

Browse files
msluszniakSS-JIA
authored andcommitted
[ET-VK] Run weight-only int8 linear on the kernel that still exists
FuseQuantizedOpsTransform rewrites a weight-only int8 linear to aten._weight_int8pack_mm, op_registry lists it as supported, so the partitioner takes it. At runtime it dispatched to linear_qcs8w_tiled / linear_qcs8w_coop, and those shaders are not in the repo: pytorch#14041 replaced them with linear_q8csw_tiled and a different suffix scheme. Every weight-only int8 model therefore aborts on its first execute with Could not find ShaderInfo with name linear_qcs8w_tiled_texture3d_texture3d_texture2d_texture2d_half_o4x3 aten._weight_int8pack_mm carries the same operands as et_vk.linear_q8csw minus the bias, so register it on that implementation and drop the 8-bit half of QuantizedLinearQCSNW.cpp, whose shaders are gone. The 4-bit et_vk.linear_qcs4w path is left untouched; it references missing shaders too, but the group-wise kernel is not a drop-in replacement for it. Tested on a Galaxy S26 Ultra (Adreno 840) with all-mpnet-base-v2 and multi-qa-mpnet-base-dot-v1 lowered through VulkanQuantizer weight-only int8, at the 382-token bound and resized down to 128. Cosine against the fp32 eager reference is 0.998280 / 0.995343 (fp16 for comparison: 0.999972). The int8 file is 133.2 MB against 217.9 MB for fp16, and per execution it runs 59 ms against 79 ms at 382 tokens, 36 vs 49 at 254, 26 vs 29 at 128 (median of 3 rounds, arm order reversed each round, warm-up discarded).
1 parent afc7957 commit f1adb8b

2 files changed

Lines changed: 40 additions & 116 deletions

File tree

‎backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.cpp‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,6 +944,40 @@ void linear_q8csw(ComputeGraph& graph, const std::vector<ValueRef>& args) {
944944
output);
945945
}
946946

947+
// aten._weight_int8pack_mm is what the AOT weight-only int8 fusion
948+
// (FuseQuantizedOpsTransform) emits. It carries the same operands as
949+
// et_vk.linear_q8csw minus the bias, so it runs through the same
950+
// implementation.
951+
void weight_int8pack_mm(
952+
ComputeGraph& graph,
953+
const std::vector<ValueRef>& args) {
954+
int32_t idx = 0;
955+
const ValueRef fp_input = args.at(idx++);
956+
const ValueRef weight_data = args.at(idx++);
957+
const ValueRef weight_scales_data = args.at(idx++);
958+
const ValueRef output = args.at(idx++);
959+
960+
const int64_t K = graph.size_at<int64_t>(-1, fp_input);
961+
962+
QuantizationConfig input_quant_config(32, kNoQuantization, {});
963+
QuantizationConfig weight_quant_config(8, kPerChannel, {K});
964+
965+
quantized_linear_impl(
966+
graph,
967+
input_quant_config,
968+
weight_quant_config,
969+
fp_input,
970+
kDummyValueRef, // input scale
971+
kDummyValueRef, // input zp
972+
weight_data,
973+
kDummyValueRef, // weight sums
974+
weight_scales_data,
975+
kDummyValueRef, // weight zeros
976+
kDummyValueRef, // group size
977+
kDummyValueRef, // bias
978+
output);
979+
}
980+
947981
void linear_dq8ca_q4gsw(
948982
ComputeGraph& graph,
949983
const std::vector<ValueRef>& args) {
@@ -982,6 +1016,7 @@ void linear_dq8ca_q4gsw(
9821016
REGISTER_OPERATORS {
9831017
VK_REGISTER_OP(et_vk.linear_q8ta_q8csw.default, linear_q8ta_q8csw);
9841018
VK_REGISTER_OP(et_vk.linear_q8csw.default, linear_q8csw);
1019+
VK_REGISTER_OP(aten._weight_int8pack_mm.default, weight_int8pack_mm);
9851020
VK_REGISTER_OP(et_vk.linear_dq8ca_q4gsw.default, linear_dq8ca_q4gsw);
9861021
}
9871022

‎backends/vulkan/runtime/graph/ops/impl/QuantizedLinearQCSNW.cpp‎

Lines changed: 5 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -16,34 +16,6 @@
1616

1717
namespace vkcompute {
1818

19-
// Custom global workgroup size function for linear_qcs8w
20-
GlobalWorkGrid linear_qcs8w_gwg(
21-
ComputeGraph* graph,
22-
const vkapi::ShaderInfo& shader,
23-
const std::vector<ArgGroup>& args,
24-
const std::vector<ValueRef>& resize_args) {
25-
(void)shader;
26-
(void)resize_args;
27-
const ValueRef out = args.at(0).refs.at(0);
28-
return graph->create_linear_gwg(
29-
utils::safe_downcast<uint64_t>(graph->numel_of(out)));
30-
}
31-
32-
// Custom local workgroup size function for linear_qcs8w
33-
LocalWorkGroup linear_qcs8w_lwg(
34-
ComputeGraph* graph,
35-
const vkapi::ShaderInfo& shader,
36-
const GlobalWorkGrid& gwg,
37-
const std::vector<ArgGroup>& args,
38-
const std::vector<ValueRef>& resize_args) {
39-
(void)graph;
40-
(void)shader;
41-
(void)gwg;
42-
(void)args;
43-
(void)resize_args;
44-
return LocalWorkGroup(64u, 1u, 1u);
45-
}
46-
4719
// Custom global workgroup size function for linear_qcsnw_tiled
4820
GlobalWorkGrid linear_qcsnw_tiled_gwg(
4921
ComputeGraph* graph,
@@ -178,81 +150,6 @@ void resize_linear_qcsnw_node(
178150
graph->virtual_resize(out, new_out_sizes);
179151
}
180152

181-
void add_linear_qcs8w_node(
182-
ComputeGraph& graph,
183-
const ValueRef mat1,
184-
const ValueRef q_mat2_data,
185-
const ValueRef scales_data,
186-
const ValueRef out) {
187-
auto viewFn = VK_GET_OP_FN("aten.view_copy.default");
188-
ValueRef mat1_W_packed = mat1;
189-
ValueRef out_W_packed = out;
190-
// Create temporary tensors to store the width packed versions of mat1 and out
191-
TmpTensor mat1_tmp(
192-
&graph, graph.sizes_of(mat1), graph.dtype_of(mat1), utils::kWidthPacked);
193-
TmpTensor out_tmp(
194-
&graph, graph.sizes_of(out), graph.dtype_of(out), utils::kWidthPacked);
195-
if (!graph.is_buffer_storage(out) &&
196-
graph.packed_dim_of(mat1) != WHCN::kWidthDim) {
197-
// Ensure mat1 is width packed
198-
mat1_W_packed = mat1_tmp;
199-
viewFn(graph, {mat1, graph.add_none(), mat1_W_packed});
200-
// Ensure out is packed correctly
201-
out_W_packed = out_tmp;
202-
}
203-
ValueRef q_mat2 = prepack_standard_hw_transposed(
204-
graph, q_mat2_data, graph.storage_type_of(out), utils::kWidthPacked);
205-
ValueRef scales = prepack_standard(
206-
graph, scales_data, graph.storage_type_of(out), utils::kWidthPacked);
207-
208-
std::string kernel_name = "linear_qcs8w";
209-
kernel_name.reserve(kShaderNameReserve);
210-
add_packed_dim_suffix(kernel_name, graph.packed_dim_of(mat1_W_packed));
211-
add_packed_dim_suffix(kernel_name, graph.packed_dim_of(q_mat2));
212-
add_dtype_suffix(kernel_name, graph.dtype_of(out_W_packed));
213-
add_storage_type_suffix(kernel_name, graph.storage_type_of(out_W_packed));
214-
215-
std::vector<PushConstantDataInfo> pcs;
216-
if (graph.is_buffer_storage(out_W_packed)) {
217-
pcs = {
218-
graph.sizes_pc_of(out_W_packed),
219-
graph.strides_pc_of(out_W_packed),
220-
graph.sizes_pc_of(mat1_W_packed),
221-
graph.strides_pc_of(mat1),
222-
graph.strides_pc_of(q_mat2),
223-
graph.strides_pc_of(scales),
224-
graph.numel_pc_of(out_W_packed)};
225-
} else {
226-
pcs = {
227-
graph.logical_limits_pc_of(out_W_packed),
228-
graph.sizes_pc_of(mat1_W_packed),
229-
graph.sizes_pc_of(q_mat2)};
230-
}
231-
232-
graph.execute_nodes().emplace_back(new DynamicDispatchNode(
233-
graph,
234-
VK_KERNEL_FROM_STR(kernel_name),
235-
linear_qcs8w_gwg,
236-
linear_qcs8w_lwg,
237-
// Inputs and Outputs
238-
{{out_W_packed, vkapi::MemoryAccessType::WRITE},
239-
{{mat1_W_packed, q_mat2, scales}, vkapi::MemoryAccessType::READ}},
240-
// Shader params buffers
241-
{},
242-
// Push Constants
243-
pcs,
244-
// Specialization Constants
245-
{},
246-
// Resize Args
247-
{},
248-
// Resizing Logic
249-
resize_linear_qcsnw_node));
250-
if (!graph.is_buffer_storage(out) &&
251-
graph.packed_dim_of(out) != WHCN::kWidthDim) {
252-
viewFn(graph, {out_W_packed, graph.add_none(), out});
253-
}
254-
}
255-
256153
void add_linear_qcsnw_tiled_node(
257154
ComputeGraph& graph,
258155
const bool use_coop_algorithm,
@@ -293,6 +190,8 @@ void add_linear_qcsnw_tiled_node(
293190
kernel_name =
294191
use_coop_algorithm ? "linear_qcs4w_coop" : "linear_qcs4w_tiled";
295192
} else {
193+
// Unreachable: linear_qcs4w is the only caller of this function and it
194+
// always passes quant_nbits == 4. Kept so the 4-bit path is not disturbed.
296195
kernel_name =
297196
use_coop_algorithm ? "linear_qcs8w_coop" : "linear_qcs8w_tiled";
298197
}
@@ -389,18 +288,6 @@ bool can_use_coop_impl(ComputeGraph& graph, const ValueRef mat1) {
389288
return (graph.size_at<int>(-2, mat1) == 1);
390289
}
391290

392-
void weight_int8pack_mm(
393-
ComputeGraph& graph,
394-
const std::vector<ValueRef>& args) {
395-
check_linear_qcsnw_args(graph, 8, args[0], args[1], args[2], args[3]);
396-
if (can_use_tiled_impl(graph, args[0], args[1], args[2], args[3])) {
397-
bool use_coop_algorithm = can_use_coop_impl(graph, args[0]);
398-
return add_linear_qcsnw_tiled_node(
399-
graph, use_coop_algorithm, 8, args[0], args[1], args[2], args[3]);
400-
}
401-
return add_linear_qcs8w_node(graph, args[0], args[1], args[2], args[3]);
402-
}
403-
404291
void linear_qcs4w(ComputeGraph& graph, const std::vector<ValueRef>& args) {
405292
check_linear_qcsnw_args(graph, 4, args[0], args[1], args[2], args[3]);
406293

@@ -411,7 +298,9 @@ void linear_qcs4w(ComputeGraph& graph, const std::vector<ValueRef>& args) {
411298
}
412299

413300
REGISTER_OPERATORS {
414-
VK_REGISTER_OP(aten._weight_int8pack_mm.default, weight_int8pack_mm);
301+
// aten._weight_int8pack_mm is registered in QuantizedLinear.cpp, on the
302+
// maintained weight-only quantized linear implementation. The 8-bit path
303+
// here dispatched to linear_qcs8w_* shaders that no longer exist.
415304
VK_REGISTER_OP(et_vk.linear_qcs4w.default, linear_qcs4w);
416305
}
417306

0 commit comments

Comments
 (0)