99#include < executorch/extension/llm/custom_ops/op_moe.h>
1010
1111#include < executorch/extension/kernel_util/make_boxed_from_unboxed_functor.h>
12+ #if defined(EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO) && \
13+ !defined (TORCHAO_PARALLEL_EXECUTORCH )
1214#include < executorch/extension/threadpool/threadpool.h>
15+ #endif
1316#include < executorch/kernels/optimized/blas/CPUBlas.h>
1417#include < executorch/runtime/kernel/kernel_includes.h>
18+ #if defined(EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO) && \
19+ !defined (TORCHAO_PARALLEL_EXECUTORCH )
1520#include < executorch/runtime/kernel/thread_parallel_interface.h>
21+ #endif
1622
1723#include < torchao/csrc/cpu/shared_kernels/internal/packed_weights_header.h>
1824#include < torchao/csrc/cpu/torch_free_kernels/weight_packing/weight_packing.h>
1925
20- #ifdef ENABLE_QUANTIZED_MOE_FFN
21- #include < torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/kernel_selector.h>
26+ #ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
27+ #include < cpuinfo.h>
28+ #include < torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/kernel_config.h>
2229#include < torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/linear_8bit_act_xbit_weight.h>
30+ #include < torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/packed_weights_format.h>
31+ #include < torchao/csrc/cpu/torch_free_kernels/aarch64/linear/channelwise_8bit_activation_groupwise_lowbit_weight/channelwise_8bit_activation_groupwise_lowbit_weight.h>
2332#include < optional> // std::nullopt, used only by the optimized aarch64 path
24- #endif // ENABLE_QUANTIZED_MOE_FFN
33+ #endif // EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
2534
2635#include < array>
2736#include < cmath>
@@ -37,6 +46,49 @@ namespace {
3746
3847using ::executorch::aten::string_view;
3948
49+ #ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
50+ template <int kWeightNbit >
51+ const torchao::ops::linear_8bit_act_xbit_weight::UKernelConfig&
52+ universal_ukernel_config () {
53+ using torchao::ops::linear_8bit_act_xbit_weight::UKernelConfig;
54+ namespace kernel = torchao::kernels::cpu::aarch64::linear::
55+ channelwise_8bit_activation_groupwise_lowbit_weight;
56+
57+ static const auto config = [] {
58+ ET_CHECK_MSG (
59+ cpuinfo_initialize () && cpuinfo_has_arm_neon_dot (),
60+ " quantized_moe_ffn optimized path requires Arm NEON dot product" );
61+ auto result = UKernelConfig::make (
62+ /* preferred_alignment=*/ 16 ,
63+ /* n_step=*/ 8 ,
64+ /* nr=*/ 8 ,
65+ /* kr=*/ 16 ,
66+ /* sr=*/ 2 ,
67+ kWeightNbit ,
68+ /* has_weight_zeros=*/ false ,
69+ /* has_bias=*/ false ,
70+ &torchao::weight_packing::packed_weights_size,
71+ &torchao::weight_packing::packed_weights_offset,
72+ &torchao::weight_packing::pack_weights<kWeightNbit , 8 , 16 , 2 >,
73+ {});
74+ result.linear_configs [0 ] = UKernelConfig::linear_config_type ({
75+ /* m_step=*/ 1 ,
76+ /* mr=*/ 1 ,
77+ &kernel::packed_activations_size,
78+ &kernel::packed_activations_offset,
79+ &kernel::pack_activations<1 , 16 , 2 >,
80+ &kernel::kernel_1x8x16_f32_neondot<
81+ kWeightNbit ,
82+ /* has_weight_zeros=*/ false ,
83+ /* has_lut=*/ false >,
84+ });
85+ result.validate ();
86+ return result;
87+ }();
88+ return config;
89+ }
90+ #endif
91+
4092// Numerically-stable sigmoid. Branching on sign keeps exp()'s argument
4193// non-positive on both sides, so it can never overflow.
4294inline float stable_sigmoid (float v) {
@@ -189,7 +241,7 @@ inline void reference_linear(
189241
190242// Dispatch a single per-expert grouped GEMM through torchao's
191243// linear_operator (optimized, aarch64) or reference unpack+dequant+gemm.
192- #ifdef ENABLE_QUANTIZED_MOE_FFN
244+ #ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
193245template <int kWeightNbit >
194246inline void torchao_linear (
195247 const uint8_t * packed_w_blob,
@@ -205,21 +257,22 @@ inline void torchao_linear(
205257 static_cast <int64_t >(torchao::ops::PackedWeightsHeader::size ()),
206258 " torchao packed blob too small to contain header" );
207259 auto header = torchao::ops::PackedWeightsHeader::read (packed_w_blob);
208- // Select the ukernel from the format declared in the header. This resolves
209- // the universal or kleidi packing automatically; a format whose kernels are
210- // not compiled into this build (e.g. kleidi when TORCHAO_ENABLE_KLEIDI is
211- // unset) throws here instead of being silently mis-read.
212- // TODO: enable KleidiAI here — build this op on xplat arm64 with
213- // -DTORCHAO_ENABLE_KLEIDI (+ -DTORCHAO_ENABLE_ARM_I8MM) and link the kleidi
214- // kernel target so a kleidi header actually resolves to a kleidi ukernel.
215- // Must be coordinated with the AoT packer emitting kleidi headers (see
216- // targets.bzl).
217- auto uk = torchao::ops::linear_8bit_act_xbit_weight::select_ukernel_config<
218- kWeightNbit >(header);
219-
220- // Validate the blob against the *selected* format's layout. nr/kr/sr and the
221- // size formula differ between universal and kleidi, so derive them from the
222- // chosen config rather than assuming a fixed layout.
260+ ET_CHECK_MSG (
261+ header.type ==
262+ torchao::ops::PackedWeightsType::
263+ linear_8bit_act_xbit_weight_universal,
264+ " quantized_moe_ffn requires universal torchao packed weights" );
265+ const auto format = torchao::ops::linear_8bit_act_xbit_weight::
266+ PackedWeightsFormat::from_packed_weights_header (header);
267+ ET_CHECK_MSG (
268+ format.weight_nbit == kWeightNbit && !format.has_weight_zeros &&
269+ !format.has_bias && format.nr == 8 && format.kr == 16 &&
270+ format.sr == 2 ,
271+ " quantized_moe_ffn received an unsupported universal weight format" );
272+ const auto & uk = universal_ukernel_config<kWeightNbit >();
273+
274+ // Validate the blob against the universal config's layout without
275+ // duplicating its packed-weight size formula.
223276 const int64_t required_bytes =
224277 static_cast <int64_t >(torchao::ops::PackedWeightsHeader::size ()) +
225278 static_cast <int64_t >(uk.packed_weights_size (
@@ -258,7 +311,7 @@ inline void torchao_linear(
258311 /* clamp_min=*/ 0 .0f ,
259312 /* clamp_max=*/ 0 .0f );
260313}
261- #endif // ENABLE_QUANTIZED_MOE_FFN
314+ #endif // EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
262315
263316inline void expert_linear_dispatch (
264317 int64_t weight_nbit,
@@ -270,12 +323,21 @@ inline void expert_linear_dispatch(
270323 int64_t k,
271324 int64_t group_size,
272325 float * out) {
273- #ifndef ENABLE_QUANTIZED_MOE_FFN
326+ #ifndef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
274327 // Reference path only: it unpacks the universal layout, so validate the blob
275328 // holds the header plus the universal packed weight-data bytes for the
276329 // claimed dims before any path dereferences it. The torchao path validates
277- // against its own selected format (universal or kleidi) inside
278- // torchao_linear.
330+ // the same required universal format inside torchao_linear.
331+ ET_CHECK_MSG (
332+ packed_blob_bytes >=
333+ static_cast <int64_t >(torchao::ops::PackedWeightsHeader::size ()),
334+ " torchao packed blob too small to contain header" );
335+ const auto header = torchao::ops::PackedWeightsHeader::read (packed_w_blob);
336+ ET_CHECK_MSG (
337+ header.type ==
338+ torchao::ops::PackedWeightsType::
339+ linear_8bit_act_xbit_weight_universal,
340+ " quantized_moe_ffn requires universal torchao packed weights" );
279341 constexpr int kNr = 8 , kKr = 16 , kSr = 2 ;
280342 const int64_t required_bytes =
281343 static_cast <int64_t >(torchao::ops::PackedWeightsHeader::size ()) +
@@ -299,10 +361,10 @@ inline void expert_linear_dispatch(
299361 static_cast <long long >(k),
300362 static_cast <long long >(group_size),
301363 static_cast <long long >(weight_nbit));
302- #endif // !ENABLE_QUANTIZED_MOE_FFN
364+ #endif // !EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
303365 switch (weight_nbit) {
304366 case 4 :
305- #ifdef ENABLE_QUANTIZED_MOE_FFN
367+ #ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
306368 torchao_linear<4 >(
307369 packed_w_blob, packed_blob_bytes, x, m, n, k, group_size, out);
308370#else
@@ -311,7 +373,7 @@ inline void expert_linear_dispatch(
311373#endif
312374 return ;
313375 case 8 :
314- #ifdef ENABLE_QUANTIZED_MOE_FFN
376+ #ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
315377 torchao_linear<8 >(
316378 packed_w_blob, packed_blob_bytes, x, m, n, k, group_size, out);
317379#else
@@ -634,26 +696,20 @@ Tensor& quantized_moe_ffn_out(
634696 }
635697 };
636698
637- #ifdef ENABLE_QUANTIZED_MOE_FFN
638- // torchao linear path (perf-sensitive). The kernel threads internally on the
639- // shared pool in the common config, or runs single-threaded when only the
640- // thread-pool-free variant is linked. Distribute experts across the pool
641- // ourselves only when the kernel won't and the pool has more than one thread
642- // -- running both would nest on one pthreadpool and deadlock.
699+ #if defined(EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO) && \
700+ !defined (TORCHAO_PARALLEL_EXECUTORCH )
643701 const bool parallelize_experts = torchao::ops::linear_8bit_act_xbit_weight::
644702 linear_operator_num_threads () == 1 &&
645703 ::executorch::extension::threadpool::get_threadpool ()
646704 ->get_thread_count() > 1;
647- #else
648- // Portable reference path: prefer simplicity over speed and run experts
649- // serially.
650- const bool parallelize_experts = false ;
651- #endif
652705 if (parallelize_experts) {
653706 torch::executor::parallel_for (0 , E, /* grain_size=*/ 1 , run_experts);
654707 } else {
655708 run_experts (0 , E);
656709 }
710+ #else
711+ run_experts (0 , E);
712+ #endif
657713
658714 // ----- 7. Weighted scatter-add unpermute (cross-expert reduction) -----
659715 // Each token sums the contributions of its top-k experts; run serially to
0 commit comments