Skip to content

Commit 6e2e55c

Browse files
authored
Use Accelerate for quantized SDPA prefill on Apple (pytorch#22760) (pytorch#22760)
Summary: Dequantize multi-row int8 Q and K into per-thread scratch on Apple and use the platform BLAS for QK as the Apple library is very performant, more so than ET's INT8 path. Decode and non-Apple platforms keep the existing int8 dot-product kernel as it beats the Eigen path. Reviewed By: digantdesai Differential Revision: D119391820
1 parent 7c4cae6 commit 6e2e55c

1 file changed

Lines changed: 79 additions & 5 deletions

File tree

‎extension/llm/custom_ops/op_sdpa_impl.h‎

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,22 @@ struct MaybeQuantizedMatrixData {
6262
dtype(dtype_) {}
6363
};
6464

65+
void dequantize_per_channel_optimized(
66+
const int8_t* in_data,
67+
const float* scales_data,
68+
const int8_t* zero_points_data,
69+
float* out_data,
70+
int64_t quant_min,
71+
int64_t quant_max,
72+
size_t outer_size,
73+
size_t in_outer_stride,
74+
size_t out_outer_stride,
75+
size_t num_channels,
76+
size_t in_channel_stride,
77+
size_t out_channel_stride,
78+
size_t channel_size,
79+
size_t qparams_stride);
80+
6581
template <typename accum_t>
6682
void _q_at_k_gemm(
6783
const int64_t q_m,
@@ -81,6 +97,55 @@ void _q_at_k_gemm(
8197
"q and k must be int8, float, half, or bfloat16");
8298
if (q_data.dtype == ScalarType::Char) {
8399
if constexpr (std::is_same<accum_t, float>::value) {
100+
if (widen_scratch != nullptr) {
101+
accum_t* k_f32 = widen_scratch;
102+
accum_t* q_f32 = widen_scratch + k_n * qk_k;
103+
dequantize_per_channel_optimized(
104+
static_cast<const int8_t*>(k_data.data),
105+
k_data.scales,
106+
k_data.zero_points,
107+
k_f32,
108+
-128,
109+
127,
110+
1,
111+
0,
112+
0,
113+
k_n,
114+
k_stride_n,
115+
qk_k,
116+
qk_k,
117+
k_data.scales_stride);
118+
dequantize_per_channel_optimized(
119+
static_cast<const int8_t*>(q_data.data),
120+
q_data.scales,
121+
q_data.zero_points,
122+
q_f32,
123+
-128,
124+
127,
125+
1,
126+
0,
127+
0,
128+
q_m,
129+
q_stride_m,
130+
qk_k,
131+
qk_k,
132+
q_data.scales_stride);
133+
::executorch::cpublas::gemm(
134+
::executorch::cpublas::TransposeType::Transpose,
135+
::executorch::cpublas::TransposeType::NoTranspose,
136+
k_n,
137+
q_m,
138+
qk_k,
139+
static_cast<accum_t>(1),
140+
k_f32,
141+
qk_k,
142+
q_f32,
143+
qk_k,
144+
static_cast<accum_t>(0),
145+
qk_data,
146+
k_n);
147+
return;
148+
}
84149
int a_stride_m_tmp, b_stride_n_tmp;
85150
auto kernel = torchao::kernels::cpu::quantized_matmul::
86151
get_int8_a_int8_b_channelwise_qmatmul(
@@ -940,16 +1005,23 @@ void cpu_flash_attention(
9401005
// Scratch for widening q@K.T to fp32 (see _q_at_k_gemm): one K block plus one
9411006
// q block. qBlockSize cannot exceed qSplitSize, so include the runtime bounds
9421007
// that determine whether any block can use the widened path.
943-
const bool widen_qk =
1008+
const bool widen_reduced_qk =
9441009
std::is_same<scalar_t, ::executorch::aten::BFloat16>::value &&
9451010
::executorch::cpublas::gemm_uses_blas() &&
9461011
headSize <= kMaxHeadSizeForWidenedQK &&
9471012
qSplitSize >= kMinQBlockForWidenedQK;
1013+
#if defined(__APPLE__)
1014+
const bool dequantize_qk = is_quantized_sdpa &&
1015+
::executorch::cpublas::gemm_uses_blas() && qSplitSize > 4;
1016+
#else
1017+
const bool dequantize_qk = false;
1018+
#endif
1019+
const bool use_qk_conversion_scratch = widen_reduced_qk || dequantize_qk;
9481020
int64_t size_per_thread_widen =
949-
widen_qk ? (kvSplitSize + qSplitSize) * headSize : 0;
1021+
use_qk_conversion_scratch ? (kvSplitSize + qSplitSize) * headSize : 0;
9501022
std::unique_ptr<char[]> allocated_buf_widen;
9511023
accum_t* widen_buf = nullptr;
952-
if (widen_qk) {
1024+
if (use_qk_conversion_scratch) {
9531025
int64_t size_widen_bytes =
9541026
size_per_thread_widen * num_thread * sizeof(accum_t);
9551027
Result<void*> scratch_widen = ctx.allocate_temp(size_widen_bytes, 64);
@@ -1109,8 +1181,10 @@ void cpu_flash_attention(
11091181
k_sub_matrix_data,
11101182
kStrideN,
11111183
qk_data,
1112-
(widen_qk && qBlockSize >= kMinQBlockForWidenedQK) ? widen_ptr
1113-
: nullptr);
1184+
((widen_reduced_qk && qBlockSize >= kMinQBlockForWidenedQK) ||
1185+
(dequantize_qk && qBlockSize > 4))
1186+
? widen_ptr
1187+
: nullptr);
11141188

11151189
// There are 4 cases that is_causal has to cover to fill
11161190
// not-attendable-position with -inf

0 commit comments

Comments
 (0)