@@ -62,6 +62,22 @@ struct MaybeQuantizedMatrixData {
6262 dtype(dtype_) {}
6363};
6464
65+ void dequantize_per_channel_optimized (
66+ const int8_t * in_data,
67+ const float * scales_data,
68+ const int8_t * zero_points_data,
69+ float * out_data,
70+ int64_t quant_min,
71+ int64_t quant_max,
72+ size_t outer_size,
73+ size_t in_outer_stride,
74+ size_t out_outer_stride,
75+ size_t num_channels,
76+ size_t in_channel_stride,
77+ size_t out_channel_stride,
78+ size_t channel_size,
79+ size_t qparams_stride);
80+
6581template <typename accum_t >
6682void _q_at_k_gemm (
6783 const int64_t q_m,
@@ -81,6 +97,55 @@ void _q_at_k_gemm(
8197 " q and k must be int8, float, half, or bfloat16" );
8298 if (q_data.dtype == ScalarType::Char) {
8399 if constexpr (std::is_same<accum_t , float >::value) {
100+ if (widen_scratch != nullptr ) {
101+ accum_t * k_f32 = widen_scratch;
102+ accum_t * q_f32 = widen_scratch + k_n * qk_k;
103+ dequantize_per_channel_optimized (
104+ static_cast <const int8_t *>(k_data.data ),
105+ k_data.scales ,
106+ k_data.zero_points ,
107+ k_f32,
108+ -128 ,
109+ 127 ,
110+ 1 ,
111+ 0 ,
112+ 0 ,
113+ k_n,
114+ k_stride_n,
115+ qk_k,
116+ qk_k,
117+ k_data.scales_stride );
118+ dequantize_per_channel_optimized (
119+ static_cast <const int8_t *>(q_data.data ),
120+ q_data.scales ,
121+ q_data.zero_points ,
122+ q_f32,
123+ -128 ,
124+ 127 ,
125+ 1 ,
126+ 0 ,
127+ 0 ,
128+ q_m,
129+ q_stride_m,
130+ qk_k,
131+ qk_k,
132+ q_data.scales_stride );
133+ ::executorch::cpublas::gemm (
134+ ::executorch::cpublas::TransposeType::Transpose,
135+ ::executorch::cpublas::TransposeType::NoTranspose,
136+ k_n,
137+ q_m,
138+ qk_k,
139+ static_cast <accum_t >(1 ),
140+ k_f32,
141+ qk_k,
142+ q_f32,
143+ qk_k,
144+ static_cast<accum_t>(0 ),
145+ qk_data,
146+ k_n);
147+ return ;
148+ }
84149 int a_stride_m_tmp, b_stride_n_tmp;
85150 auto kernel = torchao::kernels::cpu::quantized_matmul::
86151 get_int8_a_int8_b_channelwise_qmatmul (
@@ -940,16 +1005,23 @@ void cpu_flash_attention(
9401005 // Scratch for widening q@K.T to fp32 (see _q_at_k_gemm): one K block plus one
9411006 // q block. qBlockSize cannot exceed qSplitSize, so include the runtime bounds
9421007 // that determine whether any block can use the widened path.
943- const bool widen_qk =
1008+ const bool widen_reduced_qk =
9441009 std::is_same<scalar_t , ::executorch::aten::BFloat16>::value &&
9451010 ::executorch::cpublas::gemm_uses_blas () &&
9461011 headSize <= kMaxHeadSizeForWidenedQK &&
9471012 qSplitSize >= kMinQBlockForWidenedQK;
1013+ #if defined(__APPLE__)
1014+ const bool dequantize_qk = is_quantized_sdpa &&
1015+ ::executorch::cpublas::gemm_uses_blas () && qSplitSize > 4;
1016+ #else
1017+ const bool dequantize_qk = false ;
1018+ #endif
1019+ const bool use_qk_conversion_scratch = widen_reduced_qk || dequantize_qk;
9481020 int64_t size_per_thread_widen =
949- widen_qk ? (kvSplitSize + qSplitSize) * headSize : 0 ;
1021+ use_qk_conversion_scratch ? (kvSplitSize + qSplitSize) * headSize : 0 ;
9501022 std::unique_ptr<char []> allocated_buf_widen;
9511023 accum_t * widen_buf = nullptr ;
952- if (widen_qk ) {
1024+ if (use_qk_conversion_scratch ) {
9531025 int64_t size_widen_bytes =
9541026 size_per_thread_widen * num_thread * sizeof (accum_t );
9551027 Result<void *> scratch_widen = ctx.allocate_temp (size_widen_bytes, 64 );
@@ -1109,8 +1181,10 @@ void cpu_flash_attention(
11091181 k_sub_matrix_data,
11101182 kStrideN ,
11111183 qk_data,
1112- (widen_qk && qBlockSize >= kMinQBlockForWidenedQK ) ? widen_ptr
1113- : nullptr );
1184+ ((widen_reduced_qk && qBlockSize >= kMinQBlockForWidenedQK ) ||
1185+ (dequantize_qk && qBlockSize > 4 ))
1186+ ? widen_ptr
1187+ : nullptr );
11141188
11151189 // There are 4 cases that is_causal has to cover to fill
11161190 // not-attendable-position with -inf
0 commit comments