feat(xpu): support MiniMax-H3 FP8 inference (#1431)

## Summary

  - Add MiniMax-H3 FP8 inference support for Intel XPU.
  - Add FP8 configuration and launch script.
  - Port large-matrix chunking from the day0 implementation.
  - Add oneDNN primitive caching.
  - Fix missing bias in FP8 linear operations.
  - Move MMWeightFp8IntelXpu into platform-specific ops.
  - Add related tests and documentation.

  ## Validation

  - XPU extension builds successfully.
  - Python syntax checks pass.
  - git diff --check passes.

Author: qiuxin2012

configs/platforms/intel_xpu/minimax_h3_t2av_fp8.json

@@ -0,0 +1,32 @@
+{
+  "infer_steps": 30,
+  "target_video_length": 124,
+  "target_height": 544,
+  "target_width": 960,
+  "fps": 24,
+  "target_fps": 24,
+  "enable_cfg": false,
+  "cpu_offload": true,
+  "offload_granularity": "block",
+  "text_encoder_cpu_offload": true,
+  "text_encoder_offload_granularity": "block",
+  "text_encoder_release_block_offload_buffers": true,
+  "vae_cpu_offload": true,
+  "lazy_load": false,
+  "unload_modules": false,
+  "attn_type": "intel_xpu_cute_attn",
+  "rms_type": "intel_xpu",
+  "rope_type": "minimax_h3_xpu_rope",
+  "feature_caching": "NoCaching",
+  "use_compile": false,
+  "video_flow_shift": 12.0,
+  "audio_flow_shift": 3.0,
+  "vae_spatial_scale_factor": 16,
+  "audio_sampling_rate": 32000,
+  "audio_latents_per_second": 40,
+  "audio_channels": 2,
+  "keep_latents_dtype_in_scheduler": true,
+  "dit_quantized": true,
+  "dit_quant_scheme": "fp8-intel-xpu",
+  "dit_quantized_ckpt": "/llm/models/MiniMax-H3/quantized/fp8/minimax_h3_fp8_xpu.safetensors"
+}

lightx2v/common/ops/mm/mm_weight.py

@@ -124,11 +124,6 @@ def _fp8_scaled_mm(
 except ImportError:
     marlin_cuda_quant = None
 
-try:
-    import sycl_kernels
-except ImportError:
-    sycl_kernels = None
-
 
 class MMWeightTemplate(metaclass=ABCMeta):
     def __init__(
@@ -2634,64 +2629,3 @@ def apply(self, input_tensor):
 def unwrap_tp_weight(module):
     """Return the concrete tensor-owning MM implementation from a TP wrapper."""
     return module._mm if isinstance(module, MMWeightTP) else module
-
-
-@MM_WEIGHT_REGISTER("fp8-intel-xpu")
-class MMWeightFp8IntelXpu(MMWeightQuantTemplate):
-    """
-    Name: W-fp8-channel-sym-A-fp16-Intel-XPU
-
-    Intel XPU optimized FP8 kernel:
-        Weight Storage: fp8 (torch.float8_e4m3fn) - saves 50% memory
-        Computation: fp16 using PyTorch native ops
-        - Dynamically dequantize FP8 → FP16 during forward
-        - Use torch.nn.functional.linear (compatible with Intel XPU)
-
-    Benefits:
-        - Memory efficient: FP8 storage (8-bit)
-        - Compatible: FP16 compute using PyTorch native ops
-        - Intel XPU friendly: No CUDA-specific kernels
-
-    Usage in config:
-        {
-            "dit_quant_scheme": "fp8-intel-xpu",
-            "weight_auto_quant": true,
-            "dit_quantized": true
-        }
-    """
-
-    def __init__(self, weight_name, bias_name, create_cuda_buffer=False, create_cpu_buffer=False, lazy_load=False, lazy_load_file=None, is_post_adapter=False, lora_prefix=None, lora_path=""):
-        super().__init__(weight_name, bias_name, create_cuda_buffer, create_cpu_buffer, lazy_load, lazy_load_file, is_post_adapter, lora_prefix, lora_path)
-
-        self.load_func = self.load_fp8_perchannel_sym
-        self.weight_need_transpose = False  # We'll handle transpose in apply
-
-    def apply(self, input_tensor):
-        # # """
-        # Forward pass with FP8 → FP16 dequantization
-
-        # Steps:
-        # 1. Dequantize weight: fp8 → fp16 (weight * scale)
-        # 2. Compute: torch.nn.functional.linear(input_fp16, weight_fp16, bias)
-        # """
-        # # Ensure input is FP16
-        # # print(input_tensor.dtype)
-
-        if sycl_kernels is not None:
-            try:
-                return sycl_kernels.onednn_w8a16_fp8(input_tensor, self.weight, self.weight_scale.to(torch.float))
-            except RuntimeError:
-                pass  # Fall through to torch dequantization path
-
-        infer_dtype = self.infer_dtype
-        squeeze_output = False
-        if input_tensor.dim() == 3 and input_tensor.shape[0] == 1:
-            input_tensor = input_tensor.squeeze(0)
-            squeeze_output = True
-        input_tensor = input_tensor.to(infer_dtype)
-        weight_fp16 = self.weight.to(infer_dtype) * self.weight_scale.to(infer_dtype)
-        bias_fp16 = self.bias.to(infer_dtype) if hasattr(self, "bias") and self.bias is not None else None
-        output = torch.nn.functional.linear(input_tensor, weight_fp16, bias_fp16)
-        if squeeze_output:
-            output = output.unsqueeze(0)
-        return output

lightx2v/models/networks/minimax_h3/model.py

@@ -28,6 +28,7 @@
     "fp8-torchao",
     "fp8-triton",
     "fp8-vllm",
+    "fp8-intel-xpu",
     "int8-q8f",
     "int8-sgl",
     "int8-torchao",

lightx2v_kernel_xpu/README.md

@@ -19,7 +19,7 @@ Exposed as the Python package `sycl_kernels`:
 |----------|-------------|
 | `sdp(Q, K, V)` | ESIMD Flash Attention — `[B, L, H, 128]` fp16/bf16, PTL-H doubleGRF |
 | `onednn_w8a8_int8(x, qweight, scales[, bias])` | Dynamic W8A8 GEMM — rowwise INT8 activation quantization × INT8 weights |
-| `onednn_w8a16_fp8(x, qweight, scales[, bias])` | W8A16 GEMM — fp16/bf16 activations × FP8_E4M3 weights, per-column scale |
+| `onednn_w8a16_fp8(x, qweight, scales[, bias])` | Cached W8A16 GEMM — fp16/bf16/fp32 activations × FP8 E4M3/E5M2 weights, per-column scale |
 | `onednn_w4a16(x, weight, scales, zeros[, bias])` | W4A16 GEMM — fp16/bf16 activations × INT4 packed weights |
 
 Tested on **Intel Arc B390 GPU** (PTL-H / Xe2), PyTorch 2.9.1+xpu, oneAPI 2025.2.
@@ -136,13 +136,16 @@ out = sycl_kernels.onednn_w8a8_int8(x, qweight, scales)
 out = sycl_kernels.onednn_w8a8_int8(x, qweight, scales, bias)
 
 # ── W8A16 FP8 GEMM ────────────────────────────────────────────────────────────
-# x       : [M, K]  fp16 or bf16     on XPU
-# qweight : [N, K]  float8_e4m3fn    on XPU
+# x       : [M, K]  fp16/bf16/fp32   on XPU
+# qweight : [N, K]  float8_e4m3fn or float8_e5m2 on XPU
 # scales  : [N, 1]  fp32             on XPU  (per-output-channel absmax scale)
-# bias    : [N]     fp16/bf16        on XPU  (optional)
+# bias    : [N]     same dtype as x  on XPU  (optional)
 # Returns : [M, N]  same dtype as x
 out = sycl_kernels.onednn_w8a16_fp8(x, qweight, scales)
 out = sycl_kernels.onednn_w8a16_fp8(x, qweight, scales, bias)
+# Repeated shapes reuse cached oneDNN primitives. Known MiniMax-H3 large
+# projections are split along N to avoid unsupported full-size primitives.
+hits, misses, size = sycl_kernels.fp8_cache_stats()
 
 # ── W4A16 GEMM ────────────────────────────────────────────────────────────────
 # x      : [M, K]    fp16 or bf16    on XPU

lightx2v_kernel_xpu/csrc/entry.cpp

@@ -15,6 +15,7 @@
 //
 
 #include <optional>
+#include <tuple>
 #include <torch/extension.h>
 
 torch::Tensor onednn_w4a16(
@@ -39,6 +40,10 @@ torch::Tensor onednn_w8a16_fp8(
     std::optional<torch::Tensor> bias
 );
 
+void fp8_cache_clear();
+std::tuple<int64_t, int64_t, int64_t> fp8_cache_stats();
+std::tuple<int64_t, int64_t, int64_t> fp8_failure_cache_stats();
+
 torch::Tensor sdp_torch(
     torch::Tensor Q,
     torch::Tensor K,
@@ -52,9 +57,15 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
           py::arg("x"), py::arg("weight"), py::arg("scales"),
           py::arg("bias") = py::none());
     m.def("onednn_w8a16_fp8", &onednn_w8a16_fp8,
-          "onednn FP16 x FP8_E4M3 per-N-scale gemm",
+          "oneDNN W8A16 FP8 per-N-scale GEMM",
           py::arg("x"), py::arg("weight"), py::arg("scales"),
           py::arg("bias") = py::none());
+    m.def("fp8_cache_clear", &fp8_cache_clear,
+          "Clear cached oneDNN FP8 primitives and counters");
+    m.def("fp8_cache_stats", &fp8_cache_stats,
+          "Return FP8 primitive cache (hits, misses, size)");
+    m.def("fp8_failure_cache_stats", &fp8_failure_cache_stats,
+          "Return FP8 failure cache (failures, negative hits, size)");
     m.def("sdp", &sdp_torch,
           "ESIMD Flash Attention SDP [B,L,H,128] PTL-H (fp16/bf16)",
           py::arg("Q"), py::arg("K"), py::arg("V"));