Skip to content

Commit d9b3e45

Browse files
authored
CMake support for quantized MoE optimized build (pytorch#22547)
For MoE custom op
1 parent 3dad019 commit d9b3e45

6 files changed

Lines changed: 168 additions & 85 deletions

File tree

‎examples/models/llama/README.md‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -561,15 +561,14 @@ registered in `executorch.extension.llm.custom_ops.custom_ops`.
561561

562562
The runtime kernel ships in `extension/llm/custom_ops/op_moe.cpp`. It
563563
always compiles with a portable reference fallback (unpack + dequant +
564-
`cpublas::gemm`) that works on any platform. `ENABLE_QUANTIZED_MOE_FFN`
565-
is an **optimization gate**, not a correctness requirement — when
566-
defined, the kernel uses torchao's fused `linear_operator` (NEON
567-
i8mm/dotprod on aarch64) instead of the reference path.
564+
`cpublas::gemm`) that works on any platform. The optimized build option
565+
uses torchao's fused `linear_operator` (NEON i8mm/dotprod on aarch64)
566+
instead of the reference path.
568567

569568
In CMake, opt in to the optimized path with:
570569

571570
```cmake
572-
-DEXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE=ON
571+
-DEXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE_OPTIMIZED=ON
573572
```
574573

575574
In Buck, `_get_quantized_moe_deps()` in `targets.bzl` wires:

‎examples/models/llama/export_llama_lib.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -594,8 +594,8 @@ def build_args_parser() -> argparse.ArgumentParser:
594594
"Replace eager MoE feed-forward modules with the "
595595
"`llama::quantized_moe_ffn` portable-runtime custom op (INT4 "
596596
"weights, INT8 dyn-quant activations via torchao). On aarch64 "
597-
"with ENABLE_QUANTIZED_MOE_FFN the optimized torchao kernel is "
598-
"used; otherwise a portable reference fallback runs."
597+
"an optimized runtime build uses the torchao kernel; otherwise "
598+
"a portable reference fallback runs."
599599
),
600600
)
601601

‎extension/llm/custom_ops/CMakeLists.txt‎

Lines changed: 62 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ set(_common_compile_options
2020
$<$<CXX_COMPILER_ID:MSVC>:/wd4996>
2121
$<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-Wno-deprecated-declarations -fPIC>
2222
)
23-
if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64")
23+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)$")
2424
list(APPEND _common_compile_options
2525
"$<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-march=armv8.2-a+dotprod>"
2626
)
@@ -86,28 +86,72 @@ target_link_libraries(custom_ops PUBLIC ${custom_ops_libs} executorch_core)
8686
# The MoE kernel always compiles with a reference fallback (unpack + dequant +
8787
# cpublas::gemm) using the torchao weight_packing headers from third-party/ao
8888
# (already on the include path). Pass
89-
# -DEXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE=ON to additionally link the
90-
# optimized torchao linear_operator (fused INT8-dyn-act GEMM, aarch64 NEON
89+
# -DEXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE_OPTIMIZED=ON to additionally link
90+
# the optimized torchao linear_operator (fused INT8-dyn-act GEMM, aarch64 NEON
9191
# dotprod).
92-
option(EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE
92+
option(EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE_OPTIMIZED
9393
"Link the optimized torchao linear kernel for llama::quantized_moe_ffn"
9494
OFF
9595
)
96-
if(EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE)
97-
if(NOT TARGET torchao_ops_linear_8bit_act_xbit_weight_executorch)
96+
97+
function(_target_enable_quantized_moe_torchao target)
98+
target_compile_definitions(
99+
${target}
100+
PRIVATE TORCHAO_BUILD_CPU_AARCH64=1 TORCHAO_ENABLE_ARM_NEON_DOT=1
101+
TORCHAO_PARALLEL_EXECUTORCH=1
102+
TORCHAO_SHARED_KERNELS_BUILD_EXECUTORCH=1
103+
)
104+
endfunction()
105+
106+
if(EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE_OPTIMIZED)
107+
if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$")
98108
message(
99109
FATAL_ERROR
100-
"EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE=ON requires target "
101-
"torchao_ops_linear_8bit_act_xbit_weight_executorch, which is not "
102-
"defined. Build the torchao ops or set this option OFF."
110+
"EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE_OPTIMIZED=ON is supported only on "
111+
"aarch64 and arm64."
103112
)
104113
endif()
105-
# Compile definition and link must be gated on the same condition, else the
106-
# ENABLE_QUANTIZED_MOE_FFN path compiles without the library that defines it.
107-
target_compile_definitions(custom_ops PUBLIC ENABLE_QUANTIZED_MOE_FFN=1)
108-
target_link_libraries(
109-
custom_ops PUBLIC torchao_ops_linear_8bit_act_xbit_weight_executorch
114+
115+
# The selected target is also reused by custom_ops_aot_lib below.
116+
if(EXECUTORCH_BUILD_KERNELS_TORCHAO)
117+
if(NOT TARGET torchao_ops_executorch)
118+
message(FATAL_ERROR "EXECUTORCH_BUILD_KERNELS_TORCHAO=ON but target "
119+
"torchao_ops_executorch is not defined."
120+
)
121+
endif()
122+
set(quantized_moe_torchao_target torchao_ops_executorch)
123+
else()
124+
set(quantized_moe_torchao_target torchao_moe_linear)
125+
add_library(
126+
torchao_moe_linear STATIC
127+
${EXECUTORCH_ROOT}/third-party/ao/torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/linear_8bit_act_xbit_weight.cpp
128+
${EXECUTORCH_ROOT}/third-party/ao/torchao/csrc/cpu/torch_free_kernels/aarch64/quantization/quantize.cpp
129+
${EXECUTORCH_ROOT}/third-party/ao/torchao/csrc/cpu/torch_free_kernels/aarch64/reduction/compute_sum.cpp
130+
${EXECUTORCH_ROOT}/third-party/ao/torchao/csrc/cpu/torch_free_kernels/aarch64/reduction/find_min_and_max.cpp
131+
${EXECUTORCH_ROOT}/third-party/ao/torchao/csrc/cpu/torch_free_kernels/aarch64/valpacking/interleave.cpp
132+
)
133+
target_include_directories(
134+
torchao_moe_linear PRIVATE ${EXECUTORCH_ROOT}/third-party/ao
135+
)
136+
_target_enable_quantized_moe_torchao(torchao_moe_linear)
137+
target_link_libraries(
138+
torchao_moe_linear PRIVATE cpuinfo executorch_core extension_threadpool
139+
)
140+
target_compile_options(
141+
torchao_moe_linear PRIVATE ${_common_compile_options}
142+
)
143+
install(
144+
TARGETS torchao_moe_linear
145+
EXPORT ExecuTorchTargets
146+
DESTINATION ${CMAKE_INSTALL_LIBDIR}
147+
)
148+
endif()
149+
150+
target_compile_definitions(
151+
custom_ops PRIVATE EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO=1
110152
)
153+
_target_enable_quantized_moe_torchao(custom_ops)
154+
target_link_libraries(custom_ops PUBLIC ${quantized_moe_torchao_target})
111155
endif()
112156

113157
target_compile_options(custom_ops PUBLIC ${_common_compile_options})
@@ -190,21 +234,13 @@ if(EXECUTORCH_BUILD_KERNELS_LLM_AOT)
190234
custom_ops_aot_lib PUBLIC cpublas torch extension_tensor
191235
extension_threadpool
192236
)
193-
if(EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE)
194-
if(NOT TARGET torchao_ops_linear_8bit_act_xbit_weight_executorch)
195-
message(
196-
FATAL_ERROR
197-
"EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE=ON requires target "
198-
"torchao_ops_linear_8bit_act_xbit_weight_executorch, which is not "
199-
"defined. Build the torchao ops or set this option OFF."
200-
)
201-
endif()
237+
if(EXECUTORCH_BUILD_KERNELS_LLM_QUANTIZED_MOE_OPTIMIZED)
202238
target_compile_definitions(
203-
custom_ops_aot_lib PUBLIC ENABLE_QUANTIZED_MOE_FFN=1
239+
custom_ops_aot_lib PRIVATE EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO=1
204240
)
241+
_target_enable_quantized_moe_torchao(custom_ops_aot_lib)
205242
target_link_libraries(
206-
custom_ops_aot_lib
207-
PUBLIC torchao_ops_linear_8bit_act_xbit_weight_executorch
243+
custom_ops_aot_lib PUBLIC ${quantized_moe_torchao_target}
208244
)
209245
endif()
210246
if(WIN32)

‎extension/llm/custom_ops/op_moe.cpp‎

Lines changed: 93 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,28 @@
99
#include <executorch/extension/llm/custom_ops/op_moe.h>
1010

1111
#include <executorch/extension/kernel_util/make_boxed_from_unboxed_functor.h>
12+
#if defined(EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO) && \
13+
!defined(TORCHAO_PARALLEL_EXECUTORCH)
1214
#include <executorch/extension/threadpool/threadpool.h>
15+
#endif
1316
#include <executorch/kernels/optimized/blas/CPUBlas.h>
1417
#include <executorch/runtime/kernel/kernel_includes.h>
18+
#if defined(EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO) && \
19+
!defined(TORCHAO_PARALLEL_EXECUTORCH)
1520
#include <executorch/runtime/kernel/thread_parallel_interface.h>
21+
#endif
1622

1723
#include <torchao/csrc/cpu/shared_kernels/internal/packed_weights_header.h>
1824
#include <torchao/csrc/cpu/torch_free_kernels/weight_packing/weight_packing.h>
1925

20-
#ifdef ENABLE_QUANTIZED_MOE_FFN
21-
#include <torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/kernel_selector.h>
26+
#ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
27+
#include <cpuinfo.h>
28+
#include <torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/kernel_config.h>
2229
#include <torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/linear_8bit_act_xbit_weight.h>
30+
#include <torchao/csrc/cpu/shared_kernels/linear_8bit_act_xbit_weight/packed_weights_format.h>
31+
#include <torchao/csrc/cpu/torch_free_kernels/aarch64/linear/channelwise_8bit_activation_groupwise_lowbit_weight/channelwise_8bit_activation_groupwise_lowbit_weight.h>
2332
#include <optional> // std::nullopt, used only by the optimized aarch64 path
24-
#endif // ENABLE_QUANTIZED_MOE_FFN
33+
#endif // EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
2534

2635
#include <array>
2736
#include <cmath>
@@ -37,6 +46,49 @@ namespace {
3746

3847
using ::executorch::aten::string_view;
3948

49+
#ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
50+
template <int kWeightNbit>
51+
const torchao::ops::linear_8bit_act_xbit_weight::UKernelConfig&
52+
universal_ukernel_config() {
53+
using torchao::ops::linear_8bit_act_xbit_weight::UKernelConfig;
54+
namespace kernel = torchao::kernels::cpu::aarch64::linear::
55+
channelwise_8bit_activation_groupwise_lowbit_weight;
56+
57+
static const auto config = [] {
58+
ET_CHECK_MSG(
59+
cpuinfo_initialize() && cpuinfo_has_arm_neon_dot(),
60+
"quantized_moe_ffn optimized path requires Arm NEON dot product");
61+
auto result = UKernelConfig::make(
62+
/*preferred_alignment=*/16,
63+
/*n_step=*/8,
64+
/*nr=*/8,
65+
/*kr=*/16,
66+
/*sr=*/2,
67+
kWeightNbit,
68+
/*has_weight_zeros=*/false,
69+
/*has_bias=*/false,
70+
&torchao::weight_packing::packed_weights_size,
71+
&torchao::weight_packing::packed_weights_offset,
72+
&torchao::weight_packing::pack_weights<kWeightNbit, 8, 16, 2>,
73+
{});
74+
result.linear_configs[0] = UKernelConfig::linear_config_type({
75+
/*m_step=*/1,
76+
/*mr=*/1,
77+
&kernel::packed_activations_size,
78+
&kernel::packed_activations_offset,
79+
&kernel::pack_activations<1, 16, 2>,
80+
&kernel::kernel_1x8x16_f32_neondot<
81+
kWeightNbit,
82+
/*has_weight_zeros=*/false,
83+
/*has_lut=*/false>,
84+
});
85+
result.validate();
86+
return result;
87+
}();
88+
return config;
89+
}
90+
#endif
91+
4092
// Numerically-stable sigmoid. Branching on sign keeps exp()'s argument
4193
// non-positive on both sides, so it can never overflow.
4294
inline float stable_sigmoid(float v) {
@@ -189,7 +241,7 @@ inline void reference_linear(
189241

190242
// Dispatch a single per-expert grouped GEMM through torchao's
191243
// linear_operator (optimized, aarch64) or reference unpack+dequant+gemm.
192-
#ifdef ENABLE_QUANTIZED_MOE_FFN
244+
#ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
193245
template <int kWeightNbit>
194246
inline void torchao_linear(
195247
const uint8_t* packed_w_blob,
@@ -205,21 +257,22 @@ inline void torchao_linear(
205257
static_cast<int64_t>(torchao::ops::PackedWeightsHeader::size()),
206258
"torchao packed blob too small to contain header");
207259
auto header = torchao::ops::PackedWeightsHeader::read(packed_w_blob);
208-
// Select the ukernel from the format declared in the header. This resolves
209-
// the universal or kleidi packing automatically; a format whose kernels are
210-
// not compiled into this build (e.g. kleidi when TORCHAO_ENABLE_KLEIDI is
211-
// unset) throws here instead of being silently mis-read.
212-
// TODO: enable KleidiAI here — build this op on xplat arm64 with
213-
// -DTORCHAO_ENABLE_KLEIDI (+ -DTORCHAO_ENABLE_ARM_I8MM) and link the kleidi
214-
// kernel target so a kleidi header actually resolves to a kleidi ukernel.
215-
// Must be coordinated with the AoT packer emitting kleidi headers (see
216-
// targets.bzl).
217-
auto uk = torchao::ops::linear_8bit_act_xbit_weight::select_ukernel_config<
218-
kWeightNbit>(header);
219-
220-
// Validate the blob against the *selected* format's layout. nr/kr/sr and the
221-
// size formula differ between universal and kleidi, so derive them from the
222-
// chosen config rather than assuming a fixed layout.
260+
ET_CHECK_MSG(
261+
header.type ==
262+
torchao::ops::PackedWeightsType::
263+
linear_8bit_act_xbit_weight_universal,
264+
"quantized_moe_ffn requires universal torchao packed weights");
265+
const auto format = torchao::ops::linear_8bit_act_xbit_weight::
266+
PackedWeightsFormat::from_packed_weights_header(header);
267+
ET_CHECK_MSG(
268+
format.weight_nbit == kWeightNbit && !format.has_weight_zeros &&
269+
!format.has_bias && format.nr == 8 && format.kr == 16 &&
270+
format.sr == 2,
271+
"quantized_moe_ffn received an unsupported universal weight format");
272+
const auto& uk = universal_ukernel_config<kWeightNbit>();
273+
274+
// Validate the blob against the universal config's layout without
275+
// duplicating its packed-weight size formula.
223276
const int64_t required_bytes =
224277
static_cast<int64_t>(torchao::ops::PackedWeightsHeader::size()) +
225278
static_cast<int64_t>(uk.packed_weights_size(
@@ -258,7 +311,7 @@ inline void torchao_linear(
258311
/*clamp_min=*/0.0f,
259312
/*clamp_max=*/0.0f);
260313
}
261-
#endif // ENABLE_QUANTIZED_MOE_FFN
314+
#endif // EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
262315

263316
inline void expert_linear_dispatch(
264317
int64_t weight_nbit,
@@ -270,12 +323,21 @@ inline void expert_linear_dispatch(
270323
int64_t k,
271324
int64_t group_size,
272325
float* out) {
273-
#ifndef ENABLE_QUANTIZED_MOE_FFN
326+
#ifndef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
274327
// Reference path only: it unpacks the universal layout, so validate the blob
275328
// holds the header plus the universal packed weight-data bytes for the
276329
// claimed dims before any path dereferences it. The torchao path validates
277-
// against its own selected format (universal or kleidi) inside
278-
// torchao_linear.
330+
// the same required universal format inside torchao_linear.
331+
ET_CHECK_MSG(
332+
packed_blob_bytes >=
333+
static_cast<int64_t>(torchao::ops::PackedWeightsHeader::size()),
334+
"torchao packed blob too small to contain header");
335+
const auto header = torchao::ops::PackedWeightsHeader::read(packed_w_blob);
336+
ET_CHECK_MSG(
337+
header.type ==
338+
torchao::ops::PackedWeightsType::
339+
linear_8bit_act_xbit_weight_universal,
340+
"quantized_moe_ffn requires universal torchao packed weights");
279341
constexpr int kNr = 8, kKr = 16, kSr = 2;
280342
const int64_t required_bytes =
281343
static_cast<int64_t>(torchao::ops::PackedWeightsHeader::size()) +
@@ -299,10 +361,10 @@ inline void expert_linear_dispatch(
299361
static_cast<long long>(k),
300362
static_cast<long long>(group_size),
301363
static_cast<long long>(weight_nbit));
302-
#endif // !ENABLE_QUANTIZED_MOE_FFN
364+
#endif // !EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
303365
switch (weight_nbit) {
304366
case 4:
305-
#ifdef ENABLE_QUANTIZED_MOE_FFN
367+
#ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
306368
torchao_linear<4>(
307369
packed_w_blob, packed_blob_bytes, x, m, n, k, group_size, out);
308370
#else
@@ -311,7 +373,7 @@ inline void expert_linear_dispatch(
311373
#endif
312374
return;
313375
case 8:
314-
#ifdef ENABLE_QUANTIZED_MOE_FFN
376+
#ifdef EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO
315377
torchao_linear<8>(
316378
packed_w_blob, packed_blob_bytes, x, m, n, k, group_size, out);
317379
#else
@@ -634,26 +696,20 @@ Tensor& quantized_moe_ffn_out(
634696
}
635697
};
636698

637-
#ifdef ENABLE_QUANTIZED_MOE_FFN
638-
// torchao linear path (perf-sensitive). The kernel threads internally on the
639-
// shared pool in the common config, or runs single-threaded when only the
640-
// thread-pool-free variant is linked. Distribute experts across the pool
641-
// ourselves only when the kernel won't and the pool has more than one thread
642-
// -- running both would nest on one pthreadpool and deadlock.
699+
#if defined(EXECUTORCH_QUANTIZED_MOE_USE_TORCHAO) && \
700+
!defined(TORCHAO_PARALLEL_EXECUTORCH)
643701
const bool parallelize_experts = torchao::ops::linear_8bit_act_xbit_weight::
644702
linear_operator_num_threads() == 1 &&
645703
::executorch::extension::threadpool::get_threadpool()
646704
->get_thread_count() > 1;
647-
#else
648-
// Portable reference path: prefer simplicity over speed and run experts
649-
// serially.
650-
const bool parallelize_experts = false;
651-
#endif
652705
if (parallelize_experts) {
653706
torch::executor::parallel_for(0, E, /*grain_size=*/1, run_experts);
654707
} else {
655708
run_experts(0, E);
656709
}
710+
#else
711+
run_experts(0, E);
712+
#endif
657713

658714
// ----- 7. Weighted scatter-add unpermute (cross-expert reduction) -----
659715
// Each token sums the contributions of its top-k experts; run serially to

‎extension/llm/custom_ops/targets.bzl‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,13 @@ def _get_quantized_moe_preproc_flags():
5353
if runtime.is_oss:
5454
return []
5555
if is_xplat():
56-
# TODO: enable KleidiAI for the runtime here by adding
57-
# -DTORCHAO_ENABLE_KLEIDI (+ -DTORCHAO_ENABLE_ARM_I8MM=1) on arm64 and
58-
# linking the kleidi kernel target in _get_quantized_moe_deps(). The
59-
# runtime (op_moe.cpp) already selects the ukernel from the header, so
60-
# kleidi headers resolve automatically once the kernels are compiled.
61-
# Must be paired with the AoT packer emitting kleidi headers (see
62-
# _get_quantized_moe_aot_packer_deps()).
56+
# TODO: enable KleidiAI by adding its runtime config to op_moe.cpp,
57+
# compiling and linking its kernels here, and pairing it with an AoT
58+
# packer that emits Kleidi headers.
6359
return select({
6460
"DEFAULT": [],
6561
"ovr_config//cpu:arm64": [
66-
"-DENABLE_QUANTIZED_MOE_FFN",
62+
"-DEXECUTORCH_QUANTIZED_MOE_USE_TORCHAO",
6763
"-DTORCHAO_BUILD_CPU_AARCH64=1",
6864
"-DTORCHAO_ENABLE_ARM_NEON_DOT=1",
6965
],

0 commit comments

Comments
 (0)