|
1 | 1 | # SPDX-License-Identifier: Apache-2.0 |
2 | 2 | # Copyright (C) 2025-2026 FlyDSL Project Contributors |
3 | | -"""Compile + launch dispatch for the layout-API MXFP4 MoE gemm (2-stage), opus-sort. |
4 | | -
|
5 | | -This is the public entry point for the MXFP4 a4w4 / a8w4 MoE surface. It consumes |
6 | | -the standard (opus-style) sort contract emitted by ``moe_sorting_kernel`` -- |
7 | | -``sorted_token_ids`` packed ``(topk<<24)|token_id`` with sentinel ``(topk<<24)|M`` |
8 | | --- and needs NO fused-sort extras: |
9 | | -
|
10 | | - * gemm1 gathers its A rows straight from ``sorted_token_ids & 0xFFFFFF`` (padding |
11 | | - rows carry M -> the buffer-bounds load returns 0). |
12 | | - * gemm2 uses the atomic bf16 epilogue, scattering per sorted row into the output |
13 | | - via ``global.atomic.fadd`` weighted by ``sorted_weights`` -- so there is no |
14 | | - inverse-permutation (``reverse_sorted``) dependency. |
15 | | -
|
16 | | -The two ``compile_*`` builders wrap the device bodies in ``moegemm`` with the |
17 | | -``@flyc.kernel`` / ``@flyc.jit`` launch plumbing; the dtype-agnostic basics come |
18 | | -from ``utils``. |
19 | | -
|
20 | | -Covered surface (BM=32): |
21 | | - * gemm1: a4w4 + a8w4 (fp8 act), interleave + separated gate, nt/cached B-load, |
22 | | - out fp4 / fp8. |
23 | | - * gemm2: atomic epilog, a4w4 + a8w4 (fp8 intermediate). |
| 3 | +"""Compile + launch dispatch for the layout-API MXFP4 MoE gemm (BM32, opus-sort). |
| 4 | +
|
| 5 | +Public entry point for the a4w4 / a8w4 surface. Consumes the opus sort contract |
| 6 | +from ``moe_sorting_kernel`` (``sorted_token_ids`` = (topk<<24)|token_id, sentinel |
| 7 | +(topk<<24)|M); no fused-sort extras. gemm2's atomic epilogue scatters into the |
| 8 | +pre-zeroed output, so there is no reverse-permutation dependency. |
| 9 | +
|
| 10 | +The ``compile_*`` builders wrap the ``moegemm`` device bodies (@flyc.jit) in the |
| 11 | +@flyc.kernel entry + @flyc.jit launch; basics come from ``utils``. |
24 | 12 | """ |
25 | 13 |
|
26 | 14 | import flydsl.compiler as flyc |
@@ -87,8 +75,7 @@ def compile_gemm1_a4w4_port( |
87 | 75 | a_dtype="fp4", |
88 | 76 | out_dtype="fp4", |
89 | 77 | ): |
90 | | - # use_nt IS the B-load cache policy (v1's `b_aux = 2 if use_nt else 0`; |
91 | | - # tuned config BM32_NT vs BM32_CACHED): True -> nt (decode), False -> cached. |
| 78 | + # use_nt IS the B-load cache policy: True -> non-temporal, False -> cached. |
92 | 79 | b_nontemporal = use_nt |
93 | 80 | if (BM, inline_quant) != (32, False): |
94 | 81 | raise AssertionError( |
@@ -281,8 +268,8 @@ def gemm2_kernel( |
281 | 268 | aq_rsrc = buffer_ops.create_buffer_resource_from_addr(_raw(fx.Int64(arg_aq)), num_records_bytes=_aq_num) |
282 | 269 | saq = SmemPtr(allocator.get_base(), lds_off, T.i8, shape=(_aStages * _slot_bytes,)) |
283 | 270 |
|
284 | | - # Preload the first kStages K-tiles (== ALL tiles for the K_TILES<=2 fast |
285 | | - # path; == prologue for the streaming path). slot == kt for the preload. |
| 271 | + # Preload the first kStages K-tiles (all tiles for the K_TILES<=2 fast path; |
| 272 | + # the prologue for the streaming path). slot == kt for the preload. |
286 | 273 | def _issue_all_a_loads(m_row0): |
287 | 274 | for slot in range_constexpr(kStages): |
288 | 275 | _issue_a_load_lds_dt( |
|
0 commit comments